2024-12-06 05:24:37,751 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-06 05:24:37,762 main DEBUG Took 0.009146 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-06 05:24:37,762 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-06 05:24:37,762 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-06 05:24:37,763 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-06 05:24:37,764 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,770 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-06 05:24:37,781 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,782 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,782 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,783 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,784 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,785 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,786 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,786 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,787 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,787 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,788 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,788 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,788 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,789 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,789 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,789 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,789 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-06 05:24:37,790 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,790 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-06 05:24:37,791 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-06 05:24:37,792 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-06 05:24:37,794 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-06 05:24:37,794 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-06 05:24:37,795 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-06 05:24:37,796 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-06 05:24:37,803 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-06 05:24:37,806 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-06 05:24:37,807 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-06 05:24:37,807 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-06 05:24:37,808 main DEBUG createAppenders(={Console}) 2024-12-06 05:24:37,808 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-06 05:24:37,809 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-06 05:24:37,809 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-06 05:24:37,809 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-06 05:24:37,809 main DEBUG OutputStream closed 2024-12-06 05:24:37,810 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-06 05:24:37,810 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-06 05:24:37,810 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-06 05:24:37,870 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-06 05:24:37,872 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-06 05:24:37,873 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-06 05:24:37,874 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-06 05:24:37,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-06 05:24:37,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-06 05:24:37,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-06 05:24:37,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-06 05:24:37,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-06 05:24:37,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-06 05:24:37,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-06 05:24:37,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-06 05:24:37,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-06 05:24:37,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-06 05:24:37,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-06 05:24:37,877 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-06 05:24:37,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-06 05:24:37,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-06 05:24:37,880 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06 05:24:37,880 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-06 05:24:37,881 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-06 05:24:37,881 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-06T05:24:38,081 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f 2024-12-06 05:24:38,085 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-06 05:24:38,085 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-06T05:24:38,095 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-06T05:24:38,102 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-12-06T05:24:38,130 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-06T05:24:38,182 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-06T05:24:38,182 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-06T05:24:38,192 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-06T05:24:38,205 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2, deleteOnExit=true 2024-12-06T05:24:38,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-06T05:24:38,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/test.cache.data in system properties and HBase conf 2024-12-06T05:24:38,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/hadoop.tmp.dir in system properties and HBase conf 2024-12-06T05:24:38,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/hadoop.log.dir in system properties and HBase conf 2024-12-06T05:24:38,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-06T05:24:38,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-06T05:24:38,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-06T05:24:38,293 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-06T05:24:38,408 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-06T05:24:38,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-06T05:24:38,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-06T05:24:38,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-06T05:24:38,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T05:24:38,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-06T05:24:38,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-06T05:24:38,417 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-06T05:24:38,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T05:24:38,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-06T05:24:38,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/nfs.dump.dir in system properties and HBase conf 2024-12-06T05:24:38,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/java.io.tmpdir in system properties and HBase conf 2024-12-06T05:24:38,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-06T05:24:38,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-06T05:24:38,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-06T05:24:39,731 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-06T05:24:39,830 INFO [Time-limited test {}] log.Log(170): Logging initialized @2718ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-06T05:24:39,927 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:24:39,984 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T05:24:40,015 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T05:24:40,015 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T05:24:40,017 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T05:24:40,037 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:24:40,040 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/hadoop.log.dir/,AVAILABLE} 2024-12-06T05:24:40,041 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T05:24:40,246 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/java.io.tmpdir/jetty-localhost-39669-hadoop-hdfs-3_4_1-tests_jar-_-any-11249865985369767243/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T05:24:40,253 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:39669} 2024-12-06T05:24:40,253 INFO [Time-limited test {}] server.Server(415): Started @3142ms 2024-12-06T05:24:40,857 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:24:40,866 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T05:24:40,868 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T05:24:40,869 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T05:24:40,869 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T05:24:40,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/hadoop.log.dir/,AVAILABLE} 2024-12-06T05:24:40,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T05:24:40,974 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/java.io.tmpdir/jetty-localhost-44201-hadoop-hdfs-3_4_1-tests_jar-_-any-9096147682957035777/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:24:40,974 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:44201} 2024-12-06T05:24:40,974 INFO [Time-limited test {}] server.Server(415): Started @3863ms 2024-12-06T05:24:41,028 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T05:24:41,199 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:24:41,211 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T05:24:41,216 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T05:24:41,217 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T05:24:41,217 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-06T05:24:41,218 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/hadoop.log.dir/,AVAILABLE} 2024-12-06T05:24:41,219 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T05:24:41,358 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/java.io.tmpdir/jetty-localhost-46149-hadoop-hdfs-3_4_1-tests_jar-_-any-12084443307852782149/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:24:41,359 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:46149} 2024-12-06T05:24:41,360 INFO [Time-limited test {}] server.Server(415): Started @4249ms 2024-12-06T05:24:41,362 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T05:24:41,416 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-06T05:24:41,421 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-06T05:24:41,425 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-06T05:24:41,426 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-06T05:24:41,426 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-06T05:24:41,429 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/hadoop.log.dir/,AVAILABLE} 2024-12-06T05:24:41,430 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-06T05:24:41,534 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/java.io.tmpdir/jetty-localhost-40635-hadoop-hdfs-3_4_1-tests_jar-_-any-5084383607646931645/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:24:41,534 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:40635} 2024-12-06T05:24:41,535 INFO [Time-limited test {}] server.Server(415): Started @4423ms 2024-12-06T05:24:41,536 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-06T05:24:43,095 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data1/current/BP-626149022-172.17.0.3-1733462679169/current, will proceed with Du for space computation calculation, 2024-12-06T05:24:43,095 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data2/current/BP-626149022-172.17.0.3-1733462679169/current, will proceed with Du for space computation calculation, 2024-12-06T05:24:43,120 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T05:24:43,160 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a5dbb43eab4cdec with lease ID 0xf6f0412adc886d57: Processing first storage report for DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f from datanode DatanodeRegistration(127.0.0.1:43097, datanodeUuid=179633ab-ee81-4fde-98a0-466db3b1812f, infoPort=46623, infoSecurePort=0, ipcPort=46255, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169) 2024-12-06T05:24:43,161 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a5dbb43eab4cdec with lease ID 0xf6f0412adc886d57: from storage DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f node DatanodeRegistration(127.0.0.1:43097, datanodeUuid=179633ab-ee81-4fde-98a0-466db3b1812f, infoPort=46623, infoSecurePort=0, ipcPort=46255, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-06T05:24:43,161 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a5dbb43eab4cdec with lease ID 0xf6f0412adc886d57: Processing first storage report for DS-eedac7a5-7c35-4669-a222-9ad9baa8631d from datanode DatanodeRegistration(127.0.0.1:43097, datanodeUuid=179633ab-ee81-4fde-98a0-466db3b1812f, infoPort=46623, infoSecurePort=0, ipcPort=46255, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169) 2024-12-06T05:24:43,162 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a5dbb43eab4cdec with lease ID 0xf6f0412adc886d57: from storage DS-eedac7a5-7c35-4669-a222-9ad9baa8631d node DatanodeRegistration(127.0.0.1:43097, datanodeUuid=179633ab-ee81-4fde-98a0-466db3b1812f, infoPort=46623, infoSecurePort=0, ipcPort=46255, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T05:24:43,442 WARN [Thread-131 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data3/current/BP-626149022-172.17.0.3-1733462679169/current, will proceed with Du for space computation calculation, 2024-12-06T05:24:43,442 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data4/current/BP-626149022-172.17.0.3-1733462679169/current, will proceed with Du for space computation calculation, 2024-12-06T05:24:43,457 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T05:24:43,461 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb42de39f7623c4e1 with lease ID 0xf6f0412adc886d58: Processing first storage report for DS-cec6918e-88b6-4269-bc4b-a51b4702b755 from datanode DatanodeRegistration(127.0.0.1:35849, datanodeUuid=03cb825b-e297-4874-9359-89ab346321d0, infoPort=39677, infoSecurePort=0, ipcPort=40653, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169) 2024-12-06T05:24:43,461 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb42de39f7623c4e1 with lease ID 0xf6f0412adc886d58: from storage DS-cec6918e-88b6-4269-bc4b-a51b4702b755 node DatanodeRegistration(127.0.0.1:35849, datanodeUuid=03cb825b-e297-4874-9359-89ab346321d0, infoPort=39677, infoSecurePort=0, ipcPort=40653, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T05:24:43,461 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb42de39f7623c4e1 with lease ID 0xf6f0412adc886d58: Processing first storage report for DS-e9cb8bad-e551-4643-9437-6ccde66d00a7 from datanode DatanodeRegistration(127.0.0.1:35849, datanodeUuid=03cb825b-e297-4874-9359-89ab346321d0, infoPort=39677, infoSecurePort=0, ipcPort=40653, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169) 2024-12-06T05:24:43,461 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb42de39f7623c4e1 with lease ID 0xf6f0412adc886d58: from storage DS-e9cb8bad-e551-4643-9437-6ccde66d00a7 node DatanodeRegistration(127.0.0.1:35849, datanodeUuid=03cb825b-e297-4874-9359-89ab346321d0, infoPort=39677, infoSecurePort=0, ipcPort=40653, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T05:24:43,693 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data5/current/BP-626149022-172.17.0.3-1733462679169/current, will proceed with Du for space computation calculation, 2024-12-06T05:24:43,693 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data6/current/BP-626149022-172.17.0.3-1733462679169/current, will proceed with Du for space computation calculation, 2024-12-06T05:24:43,711 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-06T05:24:43,715 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe52e12c010ccf969 with lease ID 0xf6f0412adc886d59: Processing first storage report for DS-09a5675f-41c8-4c76-b091-02cb90807939 from datanode DatanodeRegistration(127.0.0.1:45599, datanodeUuid=9fdc89ee-443c-4adb-80dd-9e8967c80004, infoPort=45131, infoSecurePort=0, ipcPort=42941, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169) 2024-12-06T05:24:43,715 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe52e12c010ccf969 with lease ID 0xf6f0412adc886d59: from storage DS-09a5675f-41c8-4c76-b091-02cb90807939 node DatanodeRegistration(127.0.0.1:45599, datanodeUuid=9fdc89ee-443c-4adb-80dd-9e8967c80004, infoPort=45131, infoSecurePort=0, ipcPort=42941, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T05:24:43,716 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe52e12c010ccf969 with lease ID 0xf6f0412adc886d59: Processing first storage report for DS-0cfaf38e-be6f-4419-b97f-9f5aacee6a15 from datanode DatanodeRegistration(127.0.0.1:45599, datanodeUuid=9fdc89ee-443c-4adb-80dd-9e8967c80004, infoPort=45131, infoSecurePort=0, ipcPort=42941, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169) 2024-12-06T05:24:43,716 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe52e12c010ccf969 with lease ID 0xf6f0412adc886d59: from storage DS-0cfaf38e-be6f-4419-b97f-9f5aacee6a15 node DatanodeRegistration(127.0.0.1:45599, datanodeUuid=9fdc89ee-443c-4adb-80dd-9e8967c80004, infoPort=45131, infoSecurePort=0, ipcPort=42941, storageInfo=lv=-57;cid=testClusterID;nsid=70151135;c=1733462679169), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-06T05:24:43,777 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f 2024-12-06T05:24:43,836 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/zookeeper_0, clientPort=63857, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-06T05:24:43,843 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63857 2024-12-06T05:24:43,861 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:43,863 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:44,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741825_1001 (size=7) 2024-12-06T05:24:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741825_1001 (size=7) 2024-12-06T05:24:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741825_1001 (size=7) 2024-12-06T05:24:44,457 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727 with version=8 2024-12-06T05:24:44,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/hbase-staging 2024-12-06T05:24:44,742 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ec46afd49254:0 server-side Connection retries=45 2024-12-06T05:24:44,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:44,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:44,759 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:24:44,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:44,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:24:44,888 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-06T05:24:44,943 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-06T05:24:44,955 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-06T05:24:44,960 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:24:44,989 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 66665 (auto-detected) 2024-12-06T05:24:44,990 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-06T05:24:45,014 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43357 2024-12-06T05:24:45,042 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43357 connecting to ZooKeeper ensemble=127.0.0.1:63857 2024-12-06T05:24:45,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:433570x0, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:24:45,187 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43357-0x101a9196c580000 connected 2024-12-06T05:24:45,274 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:45,277 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:45,287 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:24:45,291 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727, hbase.cluster.distributed=false 2024-12-06T05:24:45,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:24:45,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43357 2024-12-06T05:24:45,318 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43357 2024-12-06T05:24:45,319 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43357 2024-12-06T05:24:45,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43357 2024-12-06T05:24:45,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43357 2024-12-06T05:24:45,408 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45 2024-12-06T05:24:45,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:45,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:45,410 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:24:45,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:45,410 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:24:45,413 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:24:45,415 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:24:45,416 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43053 2024-12-06T05:24:45,417 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43053 connecting to ZooKeeper ensemble=127.0.0.1:63857 2024-12-06T05:24:45,418 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:45,421 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:45,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430530x0, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:24:45,432 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:430530x0, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:24:45,432 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43053-0x101a9196c580001 connected 2024-12-06T05:24:45,435 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:24:45,443 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:24:45,445 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:24:45,451 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:24:45,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43053 2024-12-06T05:24:45,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43053 2024-12-06T05:24:45,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43053 2024-12-06T05:24:45,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43053 2024-12-06T05:24:45,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43053 2024-12-06T05:24:45,474 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45 2024-12-06T05:24:45,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:45,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:45,475 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:24:45,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:45,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:24:45,476 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:24:45,476 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:24:45,477 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42449 2024-12-06T05:24:45,480 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42449 connecting to ZooKeeper ensemble=127.0.0.1:63857 2024-12-06T05:24:45,481 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:45,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:45,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:424490x0, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:24:45,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:424490x0, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:24:45,504 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42449-0x101a9196c580002 connected 2024-12-06T05:24:45,504 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:24:45,505 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:24:45,506 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:24:45,508 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:24:45,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42449 2024-12-06T05:24:45,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42449 2024-12-06T05:24:45,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42449 2024-12-06T05:24:45,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42449 2024-12-06T05:24:45,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42449 2024-12-06T05:24:45,525 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ec46afd49254:0 server-side Connection retries=45 2024-12-06T05:24:45,525 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:45,525 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:45,525 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-06T05:24:45,525 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-06T05:24:45,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-06T05:24:45,526 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-06T05:24:45,526 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-06T05:24:45,527 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39229 2024-12-06T05:24:45,529 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39229 connecting to ZooKeeper ensemble=127.0.0.1:63857 2024-12-06T05:24:45,530 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:45,532 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:45,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:392290x0, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-06T05:24:45,547 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:392290x0, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:24:45,547 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39229-0x101a9196c580003 connected 2024-12-06T05:24:45,548 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-06T05:24:45,549 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-06T05:24:45,550 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-06T05:24:45,553 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-06T05:24:45,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39229 2024-12-06T05:24:45,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39229 2024-12-06T05:24:45,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39229 2024-12-06T05:24:45,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39229 2024-12-06T05:24:45,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39229 2024-12-06T05:24:45,573 DEBUG [M:0;ec46afd49254:43357 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ec46afd49254:43357 2024-12-06T05:24:45,574 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ec46afd49254,43357,1733462684569 2024-12-06T05:24:45,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:24:45,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:24:45,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:24:45,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:24:45,592 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ec46afd49254,43357,1733462684569 2024-12-06T05:24:45,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:24:45,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:24:45,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-06T05:24:45,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:45,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:45,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:45,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:45,633 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-06T05:24:45,634 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ec46afd49254,43357,1733462684569 from backup master directory 2024-12-06T05:24:45,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ec46afd49254,43357,1733462684569 2024-12-06T05:24:45,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:24:45,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:24:45,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:24:45,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-06T05:24:45,642 WARN [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:24:45,642 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ec46afd49254,43357,1733462684569 2024-12-06T05:24:45,644 INFO [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-06T05:24:45,645 INFO [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-06T05:24:45,704 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/hbase.id] with ID: c8507a94-304b-414d-b222-aad5346dd76a 2024-12-06T05:24:45,704 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/.tmp/hbase.id 2024-12-06T05:24:45,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741826_1002 (size=42) 2024-12-06T05:24:45,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741826_1002 (size=42) 2024-12-06T05:24:45,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741826_1002 (size=42) 2024-12-06T05:24:45,723 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/.tmp/hbase.id]:[hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/hbase.id] 2024-12-06T05:24:45,767 INFO [master/ec46afd49254:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-06T05:24:45,771 INFO [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-06T05:24:45,790 INFO [master/ec46afd49254:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-06T05:24:45,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:45,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:45,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:45,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:45,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741827_1003 (size=196) 2024-12-06T05:24:45,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741827_1003 (size=196) 2024-12-06T05:24:45,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741827_1003 (size=196) 2024-12-06T05:24:45,843 INFO [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:24:45,846 INFO [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-06T05:24:45,855 INFO [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:45,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741828_1004 (size=1189) 2024-12-06T05:24:45,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741828_1004 (size=1189) 2024-12-06T05:24:45,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741828_1004 (size=1189) 2024-12-06T05:24:45,908 INFO [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store 2024-12-06T05:24:45,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741829_1005 (size=34) 2024-12-06T05:24:45,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741829_1005 (size=34) 2024-12-06T05:24:45,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741829_1005 (size=34) 2024-12-06T05:24:45,939 INFO [master/ec46afd49254:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-06T05:24:45,943 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:45,945 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:24:45,945 INFO [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:24:45,945 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:24:45,947 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:24:45,947 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:24:45,948 INFO [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:24:45,949 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733462685944Disabling compacts and flushes for region at 1733462685944Disabling writes for close at 1733462685947 (+3 ms)Writing region close event to WAL at 1733462685948 (+1 ms)Closed at 1733462685948 2024-12-06T05:24:45,952 WARN [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/.initializing 2024-12-06T05:24:45,952 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/WALs/ec46afd49254,43357,1733462684569 2024-12-06T05:24:45,962 INFO [master/ec46afd49254:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:24:45,980 INFO [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C43357%2C1733462684569, suffix=, logDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/WALs/ec46afd49254,43357,1733462684569, archiveDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/oldWALs, maxLogs=10 2024-12-06T05:24:46,015 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/WALs/ec46afd49254,43357,1733462684569/ec46afd49254%2C43357%2C1733462684569.1733462685986, exclude list is [], retry=0 2024-12-06T05:24:46,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:24:46,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:46,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:46,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:46,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-06T05:24:46,081 INFO [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/WALs/ec46afd49254,43357,1733462684569/ec46afd49254%2C43357%2C1733462684569.1733462685986 2024-12-06T05:24:46,082 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:24:46,083 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:24:46,083 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:46,086 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,087 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-06T05:24:46,162 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:46,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:24:46,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,168 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-06T05:24:46,168 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:46,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:46,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-06T05:24:46,172 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:46,173 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:46,173 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,176 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-06T05:24:46,176 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:46,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:46,177 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,180 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,182 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,186 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,187 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,190 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:24:46,193 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-06T05:24:46,198 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:24:46,199 INFO [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74527298, jitterRate=0.11054328083992004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:24:46,204 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733462686105Initializing all the Stores at 1733462686108 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462686108Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462686109 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462686109Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462686109Cleaning up temporary data from old regions at 1733462686187 (+78 ms)Region opened successfully at 1733462686204 (+17 ms) 2024-12-06T05:24:46,206 INFO [master/ec46afd49254:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-06T05:24:46,238 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f79b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0 2024-12-06T05:24:46,267 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-06T05:24:46,276 INFO [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-06T05:24:46,276 INFO [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-06T05:24:46,279 INFO [master/ec46afd49254:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-06T05:24:46,280 INFO [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-06T05:24:46,285 INFO [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-06T05:24:46,285 INFO [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-06T05:24:46,312 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-06T05:24:46,322 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-06T05:24:46,335 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-06T05:24:46,338 INFO [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-06T05:24:46,340 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-06T05:24:46,346 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-06T05:24:46,348 INFO [master/ec46afd49254:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-06T05:24:46,352 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-06T05:24:46,356 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-06T05:24:46,358 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-06T05:24:46,367 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-06T05:24:46,388 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-06T05:24:46,399 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-06T05:24:46,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:24:46,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:24:46,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:24:46,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-06T05:24:46,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,414 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ec46afd49254,43357,1733462684569, sessionid=0x101a9196c580000, setting cluster-up flag (Was=false) 2024-12-06T05:24:46,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,473 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-06T05:24:46,478 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,43357,1733462684569 2024-12-06T05:24:46,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:46,536 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-06T05:24:46,539 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ec46afd49254,43357,1733462684569 2024-12-06T05:24:46,549 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-06T05:24:46,560 INFO [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(746): ClusterId : c8507a94-304b-414d-b222-aad5346dd76a 2024-12-06T05:24:46,560 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(746): ClusterId : c8507a94-304b-414d-b222-aad5346dd76a 2024-12-06T05:24:46,561 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(746): ClusterId : c8507a94-304b-414d-b222-aad5346dd76a 2024-12-06T05:24:46,563 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:24:46,563 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:24:46,563 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-06T05:24:46,591 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:24:46,591 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:24:46,591 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-06T05:24:46,591 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:24:46,591 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:24:46,591 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-06T05:24:46,610 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:24:46,610 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:24:46,610 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-06T05:24:46,611 DEBUG [RS:1;ec46afd49254:42449 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e9d8c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0 2024-12-06T05:24:46,612 DEBUG [RS:2;ec46afd49254:39229 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77090da1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0 2024-12-06T05:24:46,614 DEBUG [RS:0;ec46afd49254:43053 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72500125, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ec46afd49254/172.17.0.3:0 2024-12-06T05:24:46,627 DEBUG [RS:2;ec46afd49254:39229 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ec46afd49254:39229 2024-12-06T05:24:46,631 DEBUG [RS:1;ec46afd49254:42449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ec46afd49254:42449 2024-12-06T05:24:46,632 INFO [RS:1;ec46afd49254:42449 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:24:46,632 INFO [RS:2;ec46afd49254:39229 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:24:46,632 INFO [RS:2;ec46afd49254:39229 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:24:46,632 INFO [RS:1;ec46afd49254:42449 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:24:46,632 DEBUG [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:24:46,632 DEBUG [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:24:46,636 INFO [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,43357,1733462684569 with port=42449, startcode=1733462685474 2024-12-06T05:24:46,641 DEBUG [RS:0;ec46afd49254:43053 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ec46afd49254:43053 2024-12-06T05:24:46,641 INFO [RS:0;ec46afd49254:43053 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-06T05:24:46,642 INFO [RS:0;ec46afd49254:43053 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-06T05:24:46,642 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,43357,1733462684569 with port=39229, startcode=1733462685524 2024-12-06T05:24:46,642 DEBUG [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-06T05:24:46,643 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,43357,1733462684569 with port=43053, startcode=1733462685378 2024-12-06T05:24:46,645 INFO [AsyncFSWAL-0-hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData-prefix:ec46afd49254,43357,1733462684569 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-12-06T05:24:46,651 DEBUG [RS:0;ec46afd49254:43053 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:24:46,651 DEBUG [RS:2;ec46afd49254:39229 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:24:46,651 DEBUG [RS:1;ec46afd49254:42449 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-06T05:24:46,663 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-06T05:24:46,675 INFO [master/ec46afd49254:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-06T05:24:46,684 INFO [master/ec46afd49254:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-06T05:24:46,691 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35067, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:24:46,691 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47155, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:24:46,691 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47281, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-06T05:24:46,693 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ec46afd49254,43357,1733462684569 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-06T05:24:46,697 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43357 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T05:24:46,703 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:24:46,703 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:24:46,703 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43357 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T05:24:46,703 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:24:46,703 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ec46afd49254:0, corePoolSize=5, maxPoolSize=5 2024-12-06T05:24:46,704 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ec46afd49254:0, corePoolSize=10, maxPoolSize=10 2024-12-06T05:24:46,704 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,704 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ec46afd49254:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:24:46,704 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43357 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-06T05:24:46,704 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,711 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:24:46,711 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-06T05:24:46,713 INFO [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733462716713 2024-12-06T05:24:46,715 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-06T05:24:46,716 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-06T05:24:46,721 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-06T05:24:46,722 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-06T05:24:46,722 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-06T05:24:46,722 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-06T05:24:46,725 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:46,725 DEBUG [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T05:24:46,725 INFO [PEWorker-2 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-06T05:24:46,725 WARN [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T05:24:46,726 DEBUG [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T05:24:46,726 WARN [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T05:24:46,724 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:46,726 DEBUG [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-06T05:24:46,726 WARN [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-06T05:24:46,727 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-06T05:24:46,729 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-06T05:24:46,729 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-06T05:24:46,732 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-06T05:24:46,733 INFO [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-06T05:24:46,735 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462686734,5,FailOnTimeoutGroup] 2024-12-06T05:24:46,736 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462686735,5,FailOnTimeoutGroup] 2024-12-06T05:24:46,736 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:46,736 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-06T05:24:46,737 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:46,738 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:46,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741831_1007 (size=1321) 2024-12-06T05:24:46,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741831_1007 (size=1321) 2024-12-06T05:24:46,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741831_1007 (size=1321) 2024-12-06T05:24:46,778 INFO [PEWorker-2 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-06T05:24:46,778 INFO [PEWorker-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727 2024-12-06T05:24:46,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741832_1008 (size=32) 2024-12-06T05:24:46,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741832_1008 (size=32) 2024-12-06T05:24:46,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741832_1008 (size=32) 2024-12-06T05:24:46,794 DEBUG [PEWorker-2 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:46,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:24:46,801 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:24:46,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:46,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:24:46,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:24:46,806 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:24:46,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:46,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:24:46,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:24:46,811 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:24:46,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:46,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:24:46,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:24:46,817 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:24:46,817 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:46,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:24:46,819 DEBUG [PEWorker-2 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:24:46,820 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740 2024-12-06T05:24:46,821 DEBUG [PEWorker-2 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740 2024-12-06T05:24:46,824 DEBUG [PEWorker-2 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:24:46,825 DEBUG [PEWorker-2 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:24:46,826 DEBUG [PEWorker-2 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:24:46,827 INFO [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,43357,1733462684569 with port=42449, startcode=1733462685474 2024-12-06T05:24:46,827 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,43357,1733462684569 with port=43053, startcode=1733462685378 2024-12-06T05:24:46,827 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(2659): reportForDuty to master=ec46afd49254,43357,1733462684569 with port=39229, startcode=1733462685524 2024-12-06T05:24:46,829 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,43053,1733462685378 2024-12-06T05:24:46,832 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43357 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,43053,1733462685378 2024-12-06T05:24:46,834 DEBUG [PEWorker-2 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:24:46,840 DEBUG [PEWorker-2 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:24:46,841 INFO [PEWorker-2 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74359205, jitterRate=0.10803849995136261}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:24:46,843 DEBUG [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727 2024-12-06T05:24:46,843 DEBUG [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45541 2024-12-06T05:24:46,843 DEBUG [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:24:46,844 DEBUG [PEWorker-2 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733462686795Initializing all the Stores at 1733462686797 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462686797Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462686797Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462686797Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462686797Cleaning up temporary data from old regions at 1733462686825 (+28 ms)Region opened successfully at 1733462686844 (+19 ms) 2024-12-06T05:24:46,844 DEBUG [PEWorker-2 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:24:46,844 INFO [PEWorker-2 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:24:46,844 DEBUG [PEWorker-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:24:46,844 DEBUG [PEWorker-2 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:24:46,844 DEBUG [PEWorker-2 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:24:46,846 INFO [PEWorker-2 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:24:46,846 DEBUG [PEWorker-2 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733462686844Disabling compacts and flushes for region at 1733462686844Disabling writes for close at 1733462686844Writing region close event to WAL at 1733462686846 (+2 ms)Closed at 1733462686846 2024-12-06T05:24:46,846 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,39229,1733462685524 2024-12-06T05:24:46,847 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43357 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,39229,1733462685524 2024-12-06T05:24:46,850 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:24:46,850 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-06T05:24:46,851 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43357 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ec46afd49254,42449,1733462685474 2024-12-06T05:24:46,851 DEBUG [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727 2024-12-06T05:24:46,851 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43357 {}] master.ServerManager(517): Registering regionserver=ec46afd49254,42449,1733462685474 2024-12-06T05:24:46,851 DEBUG [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45541 2024-12-06T05:24:46,851 DEBUG [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:24:46,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:24:46,856 DEBUG [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727 2024-12-06T05:24:46,856 DEBUG [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45541 2024-12-06T05:24:46,856 DEBUG [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-06T05:24:46,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-06T05:24:46,867 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:24:46,870 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-06T05:24:46,920 DEBUG [RS:0;ec46afd49254:43053 {}] zookeeper.ZKUtil(111): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,43053,1733462685378 2024-12-06T05:24:46,920 WARN [RS:0;ec46afd49254:43053 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:24:46,920 INFO [RS:0;ec46afd49254:43053 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:46,921 DEBUG [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,43053,1733462685378 2024-12-06T05:24:46,922 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,43053,1733462685378] 2024-12-06T05:24:46,922 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,39229,1733462685524] 2024-12-06T05:24:46,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:24:46,936 DEBUG [RS:2;ec46afd49254:39229 {}] zookeeper.ZKUtil(111): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,39229,1733462685524 2024-12-06T05:24:46,936 WARN [RS:2;ec46afd49254:39229 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:24:46,936 INFO [RS:2;ec46afd49254:39229 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:46,936 DEBUG [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524 2024-12-06T05:24:46,936 DEBUG [RS:1;ec46afd49254:42449 {}] zookeeper.ZKUtil(111): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ec46afd49254,42449,1733462685474 2024-12-06T05:24:46,937 WARN [RS:1;ec46afd49254:42449 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-06T05:24:46,937 INFO [RS:1;ec46afd49254:42449 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:46,938 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ec46afd49254,42449,1733462685474] 2024-12-06T05:24:46,939 DEBUG [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,42449,1733462685474 2024-12-06T05:24:46,959 INFO [RS:1;ec46afd49254:42449 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:24:46,959 INFO [RS:0;ec46afd49254:43053 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:24:46,961 INFO [RS:2;ec46afd49254:39229 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-06T05:24:46,973 INFO [RS:0;ec46afd49254:43053 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:24:46,973 INFO [RS:2;ec46afd49254:39229 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:24:46,973 INFO [RS:1;ec46afd49254:42449 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-06T05:24:46,979 INFO [RS:0;ec46afd49254:43053 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:24:46,979 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:46,979 INFO [RS:2;ec46afd49254:39229 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:24:46,979 INFO [RS:1;ec46afd49254:42449 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:24:46,980 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:46,980 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:46,981 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:24:46,984 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:24:46,985 INFO [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-06T05:24:46,987 INFO [RS:0;ec46afd49254:43053 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:24:46,987 INFO [RS:1;ec46afd49254:42449 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:24:46,987 INFO [RS:2;ec46afd49254:39229 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-06T05:24:46,988 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:46,988 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:46,988 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:46,989 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,989 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:24:46,990 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:24:46,990 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0, corePoolSize=2, maxPoolSize=2 2024-12-06T05:24:46,990 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ec46afd49254:0, corePoolSize=1, maxPoolSize=1 2024-12-06T05:24:46,990 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:24:46,990 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:24:46,990 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:24:46,991 DEBUG [RS:2;ec46afd49254:39229 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:24:46,991 DEBUG [RS:1;ec46afd49254:42449 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:24:46,991 DEBUG [RS:0;ec46afd49254:43053 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ec46afd49254:0, corePoolSize=3, maxPoolSize=3 2024-12-06T05:24:47,001 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,001 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,001 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,001 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,001 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,001 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43053,1733462685378-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,002 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,42449,1733462685474-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:24:47,003 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,39229,1733462685524-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:24:47,021 WARN [ec46afd49254:43357 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-06T05:24:47,027 INFO [RS:1;ec46afd49254:42449 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:24:47,027 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:24:47,027 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-06T05:24:47,030 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,42449,1733462685474-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,030 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43053,1733462685378-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,030 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,39229,1733462685524-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,030 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,030 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,031 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,031 INFO [RS:2;ec46afd49254:39229 {}] regionserver.Replication(171): ec46afd49254,39229,1733462685524 started 2024-12-06T05:24:47,031 INFO [RS:0;ec46afd49254:43053 {}] regionserver.Replication(171): ec46afd49254,43053,1733462685378 started 2024-12-06T05:24:47,031 INFO [RS:1;ec46afd49254:42449 {}] regionserver.Replication(171): ec46afd49254,42449,1733462685474 started 2024-12-06T05:24:47,057 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,058 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,43053,1733462685378, RpcServer on ec46afd49254/172.17.0.3:43053, sessionid=0x101a9196c580001 2024-12-06T05:24:47,059 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:24:47,059 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,059 INFO [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,42449,1733462685474, RpcServer on ec46afd49254/172.17.0.3:42449, sessionid=0x101a9196c580002 2024-12-06T05:24:47,059 DEBUG [RS:0;ec46afd49254:43053 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,43053,1733462685378 2024-12-06T05:24:47,059 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,43053,1733462685378' 2024-12-06T05:24:47,059 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:24:47,059 DEBUG [RS:1;ec46afd49254:42449 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,42449,1733462685474 2024-12-06T05:24:47,059 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,42449,1733462685474' 2024-12-06T05:24:47,059 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:24:47,059 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:24:47,060 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:24:47,060 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:24:47,060 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,061 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(1482): Serving as ec46afd49254,39229,1733462685524, RpcServer on ec46afd49254/172.17.0.3:39229, sessionid=0x101a9196c580003 2024-12-06T05:24:47,061 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:24:47,061 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:24:47,061 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-06T05:24:47,061 DEBUG [RS:0;ec46afd49254:43053 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,43053,1733462685378 2024-12-06T05:24:47,061 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:24:47,061 DEBUG [RS:2;ec46afd49254:39229 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ec46afd49254,39229,1733462685524 2024-12-06T05:24:47,061 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,43053,1733462685378' 2024-12-06T05:24:47,061 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:24:47,061 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,39229,1733462685524' 2024-12-06T05:24:47,061 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:24:47,061 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-06T05:24:47,061 DEBUG [RS:1;ec46afd49254:42449 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,42449,1733462685474 2024-12-06T05:24:47,061 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,42449,1733462685474' 2024-12-06T05:24:47,061 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:24:47,062 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:24:47,062 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-06T05:24:47,062 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:24:47,069 DEBUG [RS:0;ec46afd49254:43053 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:24:47,069 DEBUG [RS:1;ec46afd49254:42449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:24:47,069 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-06T05:24:47,070 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-06T05:24:47,070 INFO [RS:1;ec46afd49254:42449 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:24:47,070 INFO [RS:0;ec46afd49254:43053 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:24:47,070 DEBUG [RS:2;ec46afd49254:39229 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ec46afd49254,39229,1733462685524 2024-12-06T05:24:47,070 INFO [RS:0;ec46afd49254:43053 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:24:47,070 INFO [RS:1;ec46afd49254:42449 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:24:47,070 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ec46afd49254,39229,1733462685524' 2024-12-06T05:24:47,070 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-06T05:24:47,071 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-06T05:24:47,071 DEBUG [RS:2;ec46afd49254:39229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-06T05:24:47,071 INFO [RS:2;ec46afd49254:39229 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-06T05:24:47,071 INFO [RS:2;ec46afd49254:39229 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-06T05:24:47,176 INFO [RS:2;ec46afd49254:39229 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:24:47,176 INFO [RS:1;ec46afd49254:42449 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:24:47,176 INFO [RS:0;ec46afd49254:43053 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:24:47,181 INFO [RS:0;ec46afd49254:43053 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C43053%2C1733462685378, suffix=, logDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,43053,1733462685378, archiveDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/oldWALs, maxLogs=32 2024-12-06T05:24:47,181 INFO [RS:1;ec46afd49254:42449 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C42449%2C1733462685474, suffix=, logDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,42449,1733462685474, archiveDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/oldWALs, maxLogs=32 2024-12-06T05:24:47,181 INFO [RS:2;ec46afd49254:39229 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C39229%2C1733462685524, suffix=, logDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524, archiveDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/oldWALs, maxLogs=32 2024-12-06T05:24:47,198 DEBUG [RS:0;ec46afd49254:43053 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,43053,1733462685378/ec46afd49254%2C43053%2C1733462685378.1733462687186, exclude list is [], retry=0 2024-12-06T05:24:47,198 DEBUG [RS:2;ec46afd49254:39229 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524/ec46afd49254%2C39229%2C1733462685524.1733462687186, exclude list is [], retry=0 2024-12-06T05:24:47,199 DEBUG [RS:1;ec46afd49254:42449 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,42449,1733462685474/ec46afd49254%2C42449%2C1733462685474.1733462687186, exclude list is [], retry=0 2024-12-06T05:24:47,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:47,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:47,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:47,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:47,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:47,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:47,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:47,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:47,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:47,210 INFO [RS:2;ec46afd49254:39229 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524/ec46afd49254%2C39229%2C1733462685524.1733462687186 2024-12-06T05:24:47,215 INFO [RS:0;ec46afd49254:43053 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,43053,1733462685378/ec46afd49254%2C43053%2C1733462685378.1733462687186 2024-12-06T05:24:47,216 DEBUG [RS:0;ec46afd49254:43053 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:24:47,216 DEBUG [RS:2;ec46afd49254:39229 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:24:47,216 INFO [RS:1;ec46afd49254:42449 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,42449,1733462685474/ec46afd49254%2C42449%2C1733462685474.1733462687186 2024-12-06T05:24:47,217 DEBUG [RS:1;ec46afd49254:42449 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:24:47,275 DEBUG [ec46afd49254:43357 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-06T05:24:47,284 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(204): Hosts are {ec46afd49254=0} racks are {/default-rack=0} 2024-12-06T05:24:47,292 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:24:47,292 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:24:47,292 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-06T05:24:47,293 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:24:47,293 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:24:47,293 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-06T05:24:47,293 INFO [ec46afd49254:43357 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:24:47,293 INFO [ec46afd49254:43357 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:24:47,293 INFO [ec46afd49254:43357 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-06T05:24:47,293 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:24:47,301 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ec46afd49254,43053,1733462685378 2024-12-06T05:24:47,306 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,43053,1733462685378, state=OPENING 2024-12-06T05:24:47,314 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-06T05:24:47,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:47,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:47,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:47,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:24:47,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:24:47,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:24:47,326 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:24:47,327 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:24:47,328 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-06T05:24:47,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,43053,1733462685378}] 2024-12-06T05:24:47,509 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:24:47,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53065, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:24:47,525 INFO [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-06T05:24:47,526 INFO [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:47,527 INFO [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-06T05:24:47,530 INFO [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ec46afd49254%2C43053%2C1733462685378.meta, suffix=.meta, logDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,43053,1733462685378, archiveDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/oldWALs, maxLogs=32 2024-12-06T05:24:47,548 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,43053,1733462685378/ec46afd49254%2C43053%2C1733462685378.meta.1733462687532.meta, exclude list is [], retry=0 2024-12-06T05:24:47,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:47,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:47,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:47,561 INFO [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,43053,1733462685378/ec46afd49254%2C43053%2C1733462685378.meta.1733462687532.meta 2024-12-06T05:24:47,564 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:24:47,565 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:24:47,566 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-06T05:24:47,569 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-06T05:24:47,574 INFO [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-06T05:24:47,578 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-06T05:24:47,579 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:47,579 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-06T05:24:47,580 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-06T05:24:47,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-06T05:24:47,584 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-06T05:24:47,584 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:47,585 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:24:47,585 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-06T05:24:47,587 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-06T05:24:47,587 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:47,588 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:24:47,588 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-06T05:24:47,590 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-06T05:24:47,590 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:47,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:24:47,591 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-06T05:24:47,592 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-06T05:24:47,592 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:47,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-06T05:24:47,593 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-06T05:24:47,595 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740 2024-12-06T05:24:47,597 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740 2024-12-06T05:24:47,600 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-06T05:24:47,600 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-06T05:24:47,600 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-06T05:24:47,603 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-06T05:24:47,605 INFO [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72407958, jitterRate=0.07896265387535095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-06T05:24:47,605 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-06T05:24:47,607 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733462687580Writing region info on filesystem at 1733462687580Initializing all the Stores at 1733462687582 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462687582Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462687583 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462687583Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733462687583Cleaning up temporary data from old regions at 1733462687600 (+17 ms)Running coprocessor post-open hooks at 1733462687605 (+5 ms)Region opened successfully at 1733462687606 (+1 ms) 2024-12-06T05:24:47,615 INFO [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733462687501 2024-12-06T05:24:47,627 DEBUG [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-06T05:24:47,627 INFO [RS_OPEN_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-06T05:24:47,629 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,43053,1733462685378 2024-12-06T05:24:47,631 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ec46afd49254,43053,1733462685378, state=OPEN 2024-12-06T05:24:47,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:24:47,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:24:47,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:24:47,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-06T05:24:47,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:24:47,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:24:47,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:24:47,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-06T05:24:47,642 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ec46afd49254,43053,1733462685378 2024-12-06T05:24:47,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-06T05:24:47,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ec46afd49254,43053,1733462685378 in 312 msec 2024-12-06T05:24:47,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-06T05:24:47,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 796 msec 2024-12-06T05:24:47,661 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-06T05:24:47,661 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-06T05:24:47,679 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:24:47,680 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,43053,1733462685378, seqNum=-1] 2024-12-06T05:24:47,701 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:24:47,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41719, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:24:47,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1420 sec 2024-12-06T05:24:47,727 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733462687726, completionTime=-1 2024-12-06T05:24:47,729 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-06T05:24:47,730 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-06T05:24:47,753 INFO [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-06T05:24:47,753 INFO [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733462747753 2024-12-06T05:24:47,753 INFO [master/ec46afd49254:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733462807753 2024-12-06T05:24:47,753 INFO [master/ec46afd49254:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-06T05:24:47,755 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-06T05:24:47,762 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43357,1733462684569-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,762 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43357,1733462684569-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,762 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43357,1733462684569-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,764 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ec46afd49254:43357, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,764 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,765 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,770 DEBUG [master/ec46afd49254:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-06T05:24:47,812 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.170sec 2024-12-06T05:24:47,814 INFO [master/ec46afd49254:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-06T05:24:47,815 INFO [master/ec46afd49254:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-06T05:24:47,816 INFO [master/ec46afd49254:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-06T05:24:47,816 INFO [master/ec46afd49254:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-06T05:24:47,816 INFO [master/ec46afd49254:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-06T05:24:47,817 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43357,1733462684569-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-06T05:24:47,817 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43357,1733462684569-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-06T05:24:47,822 DEBUG [master/ec46afd49254:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-06T05:24:47,824 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-06T05:24:47,825 INFO [master/ec46afd49254:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ec46afd49254,43357,1733462684569-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-06T05:24:47,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd494aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T05:24:47,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ec46afd49254,43357,-1 for getting cluster id 2024-12-06T05:24:47,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-06T05:24:47,882 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c8507a94-304b-414d-b222-aad5346dd76a' 2024-12-06T05:24:47,884 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-06T05:24:47,884 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c8507a94-304b-414d-b222-aad5346dd76a" 2024-12-06T05:24:47,885 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c727f33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T05:24:47,885 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ec46afd49254,43357,-1] 2024-12-06T05:24:47,887 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-06T05:24:47,889 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:24:47,890 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-06T05:24:47,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75555b5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-06T05:24:47,894 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-06T05:24:47,901 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ec46afd49254,43053,1733462685378, seqNum=-1] 2024-12-06T05:24:47,902 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:24:47,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:24:47,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ec46afd49254,43357,1733462684569 2024-12-06T05:24:47,923 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:45541/hbase 2024-12-06T05:24:47,942 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=7289 2024-12-06T05:24:47,967 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:47,970 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:47,971 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:24:47,976 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-74751289, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/hregion-74751289, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:47,989 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-74751289/hregion-74751289.1733462687977, exclude list is [], retry=0 2024-12-06T05:24:47,994 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:47,994 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:47,994 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:47,997 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-74751289/hregion-74751289.1733462687977 2024-12-06T05:24:47,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:24:47,999 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => cfaf1da26200d6316795e34551eec546, NAME => 'testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/hbase 2024-12-06T05:24:48,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741838_1014 (size=64) 2024-12-06T05:24:48,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741838_1014 (size=64) 2024-12-06T05:24:48,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741838_1014 (size=64) 2024-12-06T05:24:48,012 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:48,014 INFO [StoreOpener-cfaf1da26200d6316795e34551eec546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,016 INFO [StoreOpener-cfaf1da26200d6316795e34551eec546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfaf1da26200d6316795e34551eec546 columnFamilyName a 2024-12-06T05:24:48,016 DEBUG [StoreOpener-cfaf1da26200d6316795e34551eec546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:48,017 INFO [StoreOpener-cfaf1da26200d6316795e34551eec546-1 {}] regionserver.HStore(327): Store=cfaf1da26200d6316795e34551eec546/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:48,017 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,018 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,019 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,019 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,019 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,022 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,026 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:24:48,026 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cfaf1da26200d6316795e34551eec546; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63417752, jitterRate=-0.05500185489654541}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:24:48,027 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cfaf1da26200d6316795e34551eec546: Writing region info on filesystem at 1733462688012Initializing all the Stores at 1733462688013 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462688013Cleaning up temporary data from old regions at 1733462688020 (+7 ms)Region opened successfully at 1733462688027 (+7 ms) 2024-12-06T05:24:48,027 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cfaf1da26200d6316795e34551eec546, disabling compactions & flushes 2024-12-06T05:24:48,027 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546. 2024-12-06T05:24:48,027 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546. 2024-12-06T05:24:48,027 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546. after waiting 0 ms 2024-12-06T05:24:48,027 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546. 2024-12-06T05:24:48,028 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546. 2024-12-06T05:24:48,028 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cfaf1da26200d6316795e34551eec546: Waiting for close lock at 1733462688027Disabling compacts and flushes for region at 1733462688027Disabling writes for close at 1733462688027Writing region close event to WAL at 1733462688028 (+1 ms)Closed at 1733462688028 2024-12-06T05:24:48,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741837_1013 (size=95) 2024-12-06T05:24:48,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741837_1013 (size=95) 2024-12-06T05:24:48,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741837_1013 (size=95) 2024-12-06T05:24:48,038 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:24:48,039 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-74751289:(num 1733462687977) 2024-12-06T05:24:48,041 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-06T05:24:48,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741839_1015 (size=320) 2024-12-06T05:24:48,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741839_1015 (size=320) 2024-12-06T05:24:48,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741839_1015 (size=320) 2024-12-06T05:24:48,060 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-06T05:24:48,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741840_1016 (size=253) 2024-12-06T05:24:48,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741840_1016 (size=253) 2024-12-06T05:24:48,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741840_1016 (size=253) 2024-12-06T05:24:48,089 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1, size=320 (320bytes) 2024-12-06T05:24:48,089 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-06T05:24:48,089 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-06T05:24:48,089 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1 2024-12-06T05:24:48,094 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1 after 3ms 2024-12-06T05:24:48,099 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:48,100 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1 took 13ms 2024-12-06T05:24:48,112 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1 so closing down 2024-12-06T05:24:48,112 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:24:48,115 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-06T05:24:48,117 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000001-wal-1.temp 2024-12-06T05:24:48,118 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:24:48,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741841_1017 (size=320) 2024-12-06T05:24:48,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741841_1017 (size=320) 2024-12-06T05:24:48,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741841_1017 (size=320) 2024-12-06T05:24:48,130 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-06T05:24:48,134 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002 2024-12-06T05:24:48,138 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-12-06T05:24:48,139 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1, journal: Splitting hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1, size=320 (320bytes) at 1733462688089Finishing writing output for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1 so closing down at 1733462688112 (+23 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000001-wal-1.temp at 1733462688117 (+5 ms)3 split writer threads finished at 1733462688118 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733462688130 (+12 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002 at 1733462688134 (+4 ms)Processed 2 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1733462688139 (+5 ms) 2024-12-06T05:24:48,160 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2, size=253 (253bytes) 2024-12-06T05:24:48,160 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2 2024-12-06T05:24:48,161 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2 after 1ms 2024-12-06T05:24:48,165 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:48,166 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2 took 6ms 2024-12-06T05:24:48,169 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2 so closing down 2024-12-06T05:24:48,169 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:24:48,171 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-06T05:24:48,173 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002-wal-2.temp 2024-12-06T05:24:48,173 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:24:48,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741842_1018 (size=253) 2024-12-06T05:24:48,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741842_1018 (size=253) 2024-12-06T05:24:48,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741842_1018 (size=253) 2024-12-06T05:24:48,183 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-06T05:24:48,187 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:48,189 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-12-06T05:24:48,191 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-12-06T05:24:48,191 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2, journal: Splitting hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2, size=253 (253bytes) at 1733462688160Finishing writing output for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2 so closing down at 1733462688169 (+9 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002-wal-2.temp at 1733462688173 (+4 ms)3 split writer threads finished at 1733462688174 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733462688183 (+9 ms)Processed 1 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1733462688191 (+8 ms) 2024-12-06T05:24:48,191 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:24:48,194 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:48,206 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal.1733462688195, exclude list is [], retry=0 2024-12-06T05:24:48,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:48,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:48,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:48,214 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal.1733462688195 2024-12-06T05:24:48,215 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:24:48,215 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cfaf1da26200d6316795e34551eec546, NAME => 'testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:24:48,215 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:48,215 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,215 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,217 INFO [StoreOpener-cfaf1da26200d6316795e34551eec546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,219 INFO [StoreOpener-cfaf1da26200d6316795e34551eec546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfaf1da26200d6316795e34551eec546 columnFamilyName a 2024-12-06T05:24:48,219 DEBUG [StoreOpener-cfaf1da26200d6316795e34551eec546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:48,220 INFO [StoreOpener-cfaf1da26200d6316795e34551eec546-1 {}] regionserver.HStore(327): Store=cfaf1da26200d6316795e34551eec546/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:48,220 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,221 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,225 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,226 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002 2024-12-06T05:24:48,230 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:48,235 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002 2024-12-06T05:24:48,238 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cfaf1da26200d6316795e34551eec546 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-06T05:24:48,293 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/.tmp/a/8a72808d467546fda98c3d751d79a3e8 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733462688039/Put/seqid=0 2024-12-06T05:24:48,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741844_1020 (size=5170) 2024-12-06T05:24:48,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741844_1020 (size=5170) 2024-12-06T05:24:48,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741844_1020 (size=5170) 2024-12-06T05:24:48,307 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/.tmp/a/8a72808d467546fda98c3d751d79a3e8 2024-12-06T05:24:48,348 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/.tmp/a/8a72808d467546fda98c3d751d79a3e8 as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/a/8a72808d467546fda98c3d751d79a3e8 2024-12-06T05:24:48,358 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/a/8a72808d467546fda98c3d751d79a3e8, entries=2, sequenceid=2, filesize=5.0 K 2024-12-06T05:24:48,363 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for cfaf1da26200d6316795e34551eec546 in 124ms, sequenceid=2, compaction requested=false; wal=null 2024-12-06T05:24:48,365 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/0000000000000000002 2024-12-06T05:24:48,366 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,366 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,370 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cfaf1da26200d6316795e34551eec546 2024-12-06T05:24:48,374 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaf1da26200d6316795e34551eec546/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-06T05:24:48,375 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cfaf1da26200d6316795e34551eec546; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74051956, jitterRate=0.10346013307571411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:24:48,376 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cfaf1da26200d6316795e34551eec546: Writing region info on filesystem at 1733462688215Initializing all the Stores at 1733462688217 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462688217Obtaining lock to block concurrent updates at 1733462688238 (+21 ms)Preparing flush snapshotting stores in cfaf1da26200d6316795e34551eec546 at 1733462688238Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733462688241 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733462687967.cfaf1da26200d6316795e34551eec546. at 1733462688241Flushing cfaf1da26200d6316795e34551eec546/a: creating writer at 1733462688243 (+2 ms)Flushing cfaf1da26200d6316795e34551eec546/a: appending metadata at 1733462688282 (+39 ms)Flushing cfaf1da26200d6316795e34551eec546/a: closing flushed file at 1733462688285 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7becffb3: reopening flushed file at 1733462688346 (+61 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for cfaf1da26200d6316795e34551eec546 in 124ms, sequenceid=2, compaction requested=false; wal=null at 1733462688364 (+18 ms)Cleaning up temporary data from old regions at 1733462688366 (+2 ms)Region opened successfully at 1733462688376 (+10 ms) 2024-12-06T05:24:48,403 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=375 (was 364) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:55756 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:45541/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:35824 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:54916 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:35636 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:55710 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:55076 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=279 (was 279), ProcessCount=11 (was 11), AvailableMemoryMB=7287 (was 7289) 2024-12-06T05:24:48,414 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=375, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=7286 2024-12-06T05:24:48,428 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:48,430 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:48,431 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:24:48,435 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95229018, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/hregion-95229018, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:48,446 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95229018/hregion-95229018.1733462688435, exclude list is [], retry=0 2024-12-06T05:24:48,450 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:48,451 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:48,452 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:48,456 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95229018/hregion-95229018.1733462688435 2024-12-06T05:24:48,456 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:24:48,456 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5421f10ccab890e88b535b5bcc5d04f0, NAME => 'testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/hbase 2024-12-06T05:24:48,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741846_1022 (size=64) 2024-12-06T05:24:48,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741846_1022 (size=64) 2024-12-06T05:24:48,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741846_1022 (size=64) 2024-12-06T05:24:48,471 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:48,474 INFO [StoreOpener-5421f10ccab890e88b535b5bcc5d04f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,477 INFO [StoreOpener-5421f10ccab890e88b535b5bcc5d04f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5421f10ccab890e88b535b5bcc5d04f0 columnFamilyName a 2024-12-06T05:24:48,477 DEBUG [StoreOpener-5421f10ccab890e88b535b5bcc5d04f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:48,478 INFO [StoreOpener-5421f10ccab890e88b535b5bcc5d04f0-1 {}] regionserver.HStore(327): Store=5421f10ccab890e88b535b5bcc5d04f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:48,478 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,479 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,480 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,480 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,481 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,483 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,487 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:24:48,487 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5421f10ccab890e88b535b5bcc5d04f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70429283, jitterRate=0.04947809875011444}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:24:48,488 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5421f10ccab890e88b535b5bcc5d04f0: Writing region info on filesystem at 1733462688471Initializing all the Stores at 1733462688474 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462688474Cleaning up temporary data from old regions at 1733462688481 (+7 ms)Region opened successfully at 1733462688488 (+7 ms) 2024-12-06T05:24:48,488 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5421f10ccab890e88b535b5bcc5d04f0, disabling compactions & flushes 2024-12-06T05:24:48,488 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0. 2024-12-06T05:24:48,488 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0. 2024-12-06T05:24:48,488 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0. after waiting 0 ms 2024-12-06T05:24:48,488 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0. 2024-12-06T05:24:48,489 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0. 2024-12-06T05:24:48,489 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5421f10ccab890e88b535b5bcc5d04f0: Waiting for close lock at 1733462688488Disabling compacts and flushes for region at 1733462688488Disabling writes for close at 1733462688488Writing region close event to WAL at 1733462688489 (+1 ms)Closed at 1733462688489 2024-12-06T05:24:48,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741845_1021 (size=95) 2024-12-06T05:24:48,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741845_1021 (size=95) 2024-12-06T05:24:48,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741845_1021 (size=95) 2024-12-06T05:24:48,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:24:48,495 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95229018:(num 1733462688435) 2024-12-06T05:24:48,496 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-06T05:24:48,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741847_1023 (size=320) 2024-12-06T05:24:48,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741847_1023 (size=320) 2024-12-06T05:24:48,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741847_1023 (size=320) 2024-12-06T05:24:48,510 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-06T05:24:48,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741848_1024 (size=253) 2024-12-06T05:24:48,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741848_1024 (size=253) 2024-12-06T05:24:48,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741848_1024 (size=253) 2024-12-06T05:24:48,536 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2, size=253 (253bytes) 2024-12-06T05:24:48,536 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2 2024-12-06T05:24:48,537 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2 after 1ms 2024-12-06T05:24:48,540 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:48,540 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2 took 5ms 2024-12-06T05:24:48,543 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2 so closing down 2024-12-06T05:24:48,543 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:24:48,546 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-06T05:24:48,548 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002-wal-2.temp 2024-12-06T05:24:48,549 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:24:48,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741849_1025 (size=253) 2024-12-06T05:24:48,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741849_1025 (size=253) 2024-12-06T05:24:48,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741849_1025 (size=253) 2024-12-06T05:24:48,561 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-06T05:24:48,564 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002 2024-12-06T05:24:48,564 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-12-06T05:24:48,564 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2, journal: Splitting hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2, size=253 (253bytes) at 1733462688536Finishing writing output for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2 so closing down at 1733462688543 (+7 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002-wal-2.temp at 1733462688549 (+6 ms)3 split writer threads finished at 1733462688549Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733462688561 (+12 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002 at 1733462688564 (+3 ms)Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1733462688564 2024-12-06T05:24:48,579 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1, size=320 (320bytes) 2024-12-06T05:24:48,579 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1 2024-12-06T05:24:48,580 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1 after 1ms 2024-12-06T05:24:48,584 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:48,584 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1 took 6ms 2024-12-06T05:24:48,587 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1 so closing down 2024-12-06T05:24:48,588 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:24:48,590 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-06T05:24:48,592 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000001-wal-1.temp 2024-12-06T05:24:48,592 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:24:48,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741850_1026 (size=320) 2024-12-06T05:24:48,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741850_1026 (size=320) 2024-12-06T05:24:48,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741850_1026 (size=320) 2024-12-06T05:24:48,601 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-06T05:24:48,605 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:48,608 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002, length=253 2024-12-06T05:24:48,611 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002 2024-12-06T05:24:48,611 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-12-06T05:24:48,611 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1, journal: Splitting hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1, size=320 (320bytes) at 1733462688579Finishing writing output for hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1 so closing down at 1733462688587 (+8 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000001-wal-1.temp at 1733462688592 (+5 ms)3 split writer threads finished at 1733462688592Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733462688601 (+9 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002 at 1733462688611 (+10 ms)Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1733462688611 2024-12-06T05:24:48,612 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:24:48,614 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:48,626 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal.1733462688615, exclude list is [], retry=0 2024-12-06T05:24:48,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:48,631 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:48,631 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:48,634 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal.1733462688615 2024-12-06T05:24:48,634 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:24:48,634 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5421f10ccab890e88b535b5bcc5d04f0, NAME => 'testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:24:48,634 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:48,634 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,635 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,637 INFO [StoreOpener-5421f10ccab890e88b535b5bcc5d04f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,638 INFO [StoreOpener-5421f10ccab890e88b535b5bcc5d04f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5421f10ccab890e88b535b5bcc5d04f0 columnFamilyName a 2024-12-06T05:24:48,639 DEBUG [StoreOpener-5421f10ccab890e88b535b5bcc5d04f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:48,639 INFO [StoreOpener-5421f10ccab890e88b535b5bcc5d04f0-1 {}] regionserver.HStore(327): Store=5421f10ccab890e88b535b5bcc5d04f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:48,640 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,641 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,644 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002 2024-12-06T05:24:48,647 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:48,649 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002 2024-12-06T05:24:48,649 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5421f10ccab890e88b535b5bcc5d04f0 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-06T05:24:48,671 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/.tmp/a/0d5c4c1668d34d909bcf6edd9b1b10ed is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733462688495/Put/seqid=0 2024-12-06T05:24:48,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741852_1028 (size=5170) 2024-12-06T05:24:48,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741852_1028 (size=5170) 2024-12-06T05:24:48,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741852_1028 (size=5170) 2024-12-06T05:24:48,687 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/.tmp/a/0d5c4c1668d34d909bcf6edd9b1b10ed 2024-12-06T05:24:48,699 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/.tmp/a/0d5c4c1668d34d909bcf6edd9b1b10ed as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/a/0d5c4c1668d34d909bcf6edd9b1b10ed 2024-12-06T05:24:48,710 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/a/0d5c4c1668d34d909bcf6edd9b1b10ed, entries=2, sequenceid=2, filesize=5.0 K 2024-12-06T05:24:48,710 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 5421f10ccab890e88b535b5bcc5d04f0 in 61ms, sequenceid=2, compaction requested=false; wal=null 2024-12-06T05:24:48,712 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/0000000000000000002 2024-12-06T05:24:48,713 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,713 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,717 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5421f10ccab890e88b535b5bcc5d04f0 2024-12-06T05:24:48,721 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/5421f10ccab890e88b535b5bcc5d04f0/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-06T05:24:48,722 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5421f10ccab890e88b535b5bcc5d04f0; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63883608, jitterRate=-0.048060059547424316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:24:48,723 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5421f10ccab890e88b535b5bcc5d04f0: Writing region info on filesystem at 1733462688635Initializing all the Stores at 1733462688636 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462688636Obtaining lock to block concurrent updates at 1733462688650 (+14 ms)Preparing flush snapshotting stores in 5421f10ccab890e88b535b5bcc5d04f0 at 1733462688650Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733462688650Flushing stores of testReplayEditsWrittenIntoWAL,,1733462688428.5421f10ccab890e88b535b5bcc5d04f0. at 1733462688650Flushing 5421f10ccab890e88b535b5bcc5d04f0/a: creating writer at 1733462688650Flushing 5421f10ccab890e88b535b5bcc5d04f0/a: appending metadata at 1733462688669 (+19 ms)Flushing 5421f10ccab890e88b535b5bcc5d04f0/a: closing flushed file at 1733462688669Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c0c9632: reopening flushed file at 1733462688697 (+28 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 5421f10ccab890e88b535b5bcc5d04f0 in 61ms, sequenceid=2, compaction requested=false; wal=null at 1733462688710 (+13 ms)Cleaning up temporary data from old regions at 1733462688713 (+3 ms)Region opened successfully at 1733462688722 (+9 ms) 2024-12-06T05:24:48,743 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=385 (was 375) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:55826 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:55162 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:35636 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:55710 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:35890 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=279 (was 279), ProcessCount=11 (was 11), AvailableMemoryMB=7267 (was 7286) 2024-12-06T05:24:48,753 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=7265 2024-12-06T05:24:48,770 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:48,773 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:48,773 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:24:48,777 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-65810008, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/hregion-65810008, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:48,791 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-65810008/hregion-65810008.1733462688778, exclude list is [], retry=0 2024-12-06T05:24:48,796 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:48,796 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:48,796 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:48,798 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-65810008/hregion-65810008.1733462688778 2024-12-06T05:24:48,799 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:24:48,799 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 032405e2aa1e6e2fbc4fb6bf85f992b0, NAME => 'testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/hbase 2024-12-06T05:24:48,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741854_1030 (size=64) 2024-12-06T05:24:48,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741854_1030 (size=64) 2024-12-06T05:24:48,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741854_1030 (size=64) 2024-12-06T05:24:48,814 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:48,816 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:48,818 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 032405e2aa1e6e2fbc4fb6bf85f992b0 columnFamilyName a 2024-12-06T05:24:48,818 DEBUG [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:48,819 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(327): Store=032405e2aa1e6e2fbc4fb6bf85f992b0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:48,819 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:48,821 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 032405e2aa1e6e2fbc4fb6bf85f992b0 columnFamilyName b 2024-12-06T05:24:48,821 DEBUG [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:48,822 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(327): Store=032405e2aa1e6e2fbc4fb6bf85f992b0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:48,822 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:48,824 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 032405e2aa1e6e2fbc4fb6bf85f992b0 columnFamilyName c 2024-12-06T05:24:48,824 DEBUG [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:48,825 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(327): Store=032405e2aa1e6e2fbc4fb6bf85f992b0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:48,825 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:48,826 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:48,827 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:48,828 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:48,828 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:48,829 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:24:48,830 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:48,833 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:24:48,834 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 032405e2aa1e6e2fbc4fb6bf85f992b0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70992239, jitterRate=0.05786679685115814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:24:48,834 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 032405e2aa1e6e2fbc4fb6bf85f992b0: Writing region info on filesystem at 1733462688814Initializing all the Stores at 1733462688815 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462688815Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462688816 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462688816Cleaning up temporary data from old regions at 1733462688828 (+12 ms)Region opened successfully at 1733462688834 (+6 ms) 2024-12-06T05:24:48,834 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 032405e2aa1e6e2fbc4fb6bf85f992b0, disabling compactions & flushes 2024-12-06T05:24:48,835 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. 2024-12-06T05:24:48,835 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. 2024-12-06T05:24:48,835 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. after waiting 0 ms 2024-12-06T05:24:48,835 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. 2024-12-06T05:24:48,835 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. 2024-12-06T05:24:48,835 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 032405e2aa1e6e2fbc4fb6bf85f992b0: Waiting for close lock at 1733462688834Disabling compacts and flushes for region at 1733462688834Disabling writes for close at 1733462688835 (+1 ms)Writing region close event to WAL at 1733462688835Closed at 1733462688835 2024-12-06T05:24:48,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741853_1029 (size=95) 2024-12-06T05:24:48,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741853_1029 (size=95) 2024-12-06T05:24:48,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741853_1029 (size=95) 2024-12-06T05:24:48,842 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:24:48,843 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-65810008:(num 1733462688778) 2024-12-06T05:24:48,843 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:24:48,846 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:48,861 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846, exclude list is [], retry=0 2024-12-06T05:24:48,865 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:48,866 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:48,866 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:48,870 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 2024-12-06T05:24:48,872 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:24:49,154 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846, size=0 (0bytes) 2024-12-06T05:24:49,154 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 might be still open, length is 0 2024-12-06T05:24:49,154 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 2024-12-06T05:24:49,155 WARN [IPC Server handler 1 on default port 45541 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-06T05:24:49,155 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 after 1ms 2024-12-06T05:24:52,147 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:55194 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43097:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55194 dst: /127.0.0.1:43097 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43097 remote=/127.0.0.1:55194]. Total timeout mills is 60000, 56964 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:24:52,149 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:35922 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45599:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35922 dst: /127.0.0.1:45599 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:24:52,149 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:55846 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:35849:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55846 dst: /127.0.0.1:35849 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:24:52,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741855_1032 (size=263633) 2024-12-06T05:24:52,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741855_1032 (size=263633) 2024-12-06T05:24:52,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741855_1032 (size=263633) 2024-12-06T05:24:53,094 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T05:24:53,145 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-06T05:24:53,156 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 after 4002ms 2024-12-06T05:24:53,161 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:53,163 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 took 4009ms 2024-12-06T05:24:53,168 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733462688846.temp 2024-12-06T05:24:53,176 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000000001-wal.1733462688846.temp 2024-12-06T05:24:53,299 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846; continuing. 2024-12-06T05:24:53,299 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 so closing down 2024-12-06T05:24:53,299 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:24:53,302 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:24:53,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741856_1033 (size=263641) 2024-12-06T05:24:53,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741856_1033 (size=263641) 2024-12-06T05:24:53,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741856_1033 (size=263641) 2024-12-06T05:24:53,307 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000000001-wal.1733462688846.temp (wrote 3002 edits, skipped 0 edits in 69 ms) 2024-12-06T05:24:53,310 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000000001-wal.1733462688846.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000003002 2024-12-06T05:24:53,310 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 147 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846, size=0, length=0, corrupted=false, cancelled=false 2024-12-06T05:24:53,310 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846, journal: Splitting hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846, size=0 (0bytes) at 1733462689154Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000000001-wal.1733462688846.temp at 1733462693176 (+4022 ms)Split 1024 edits, skipped 0 edits. at 1733462693235 (+59 ms)Split 2048 edits, skipped 0 edits. at 1733462693269 (+34 ms)Finishing writing output for hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 so closing down at 1733462693299 (+30 ms)3 split writer threads finished at 1733462693302 (+3 ms)Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000000001-wal.1733462688846.temp (wrote 3002 edits, skipped 0 edits in 69 ms) at 1733462693307 (+5 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000000001-wal.1733462688846.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000003002 at 1733462693310 (+3 ms)Processed 3002 edits across 1 Regions in 147 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846, size=0, length=0, corrupted=false, cancelled=false at 1733462693310 2024-12-06T05:24:53,315 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 to hdfs://localhost:45541/hbase/oldWALs/wal.1733462688846 2024-12-06T05:24:53,319 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000003002 2024-12-06T05:24:53,319 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:24:53,323 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:53,343 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462693324, exclude list is [], retry=0 2024-12-06T05:24:53,347 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:53,347 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:53,348 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:53,353 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462693324 2024-12-06T05:24:53,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:24:53,353 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:53,356 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:53,358 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 032405e2aa1e6e2fbc4fb6bf85f992b0 columnFamilyName a 2024-12-06T05:24:53,358 DEBUG [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:53,359 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(327): Store=032405e2aa1e6e2fbc4fb6bf85f992b0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:53,359 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:53,361 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 032405e2aa1e6e2fbc4fb6bf85f992b0 columnFamilyName b 2024-12-06T05:24:53,361 DEBUG [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:53,362 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(327): Store=032405e2aa1e6e2fbc4fb6bf85f992b0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:53,362 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:53,364 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 032405e2aa1e6e2fbc4fb6bf85f992b0 columnFamilyName c 2024-12-06T05:24:53,364 DEBUG [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:53,365 INFO [StoreOpener-032405e2aa1e6e2fbc4fb6bf85f992b0-1 {}] regionserver.HStore(327): Store=032405e2aa1e6e2fbc4fb6bf85f992b0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:53,365 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:53,366 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:53,368 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:53,369 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000003002 2024-12-06T05:24:53,372 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:53,405 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-06T05:24:53,767 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 032405e2aa1e6e2fbc4fb6bf85f992b0 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-06T05:24:53,815 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/a/a00f5ef2775b45869b793b1e65ae515b is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733462688879/Put/seqid=0 2024-12-06T05:24:53,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741858_1035 (size=50463) 2024-12-06T05:24:53,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741858_1035 (size=50463) 2024-12-06T05:24:53,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741858_1035 (size=50463) 2024-12-06T05:24:53,836 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/a/a00f5ef2775b45869b793b1e65ae515b 2024-12-06T05:24:53,847 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/a/a00f5ef2775b45869b793b1e65ae515b as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/a/a00f5ef2775b45869b793b1e65ae515b 2024-12-06T05:24:53,854 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/a/a00f5ef2775b45869b793b1e65ae515b, entries=754, sequenceid=754, filesize=49.3 K 2024-12-06T05:24:53,855 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 032405e2aa1e6e2fbc4fb6bf85f992b0 in 87ms, sequenceid=754, compaction requested=false; wal=null 2024-12-06T05:24:53,892 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-06T05:24:53,892 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 032405e2aa1e6e2fbc4fb6bf85f992b0 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-06T05:24:53,904 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/a/13ed9d6beb854f8f982472cee9b14b6d is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733462688927/Put/seqid=0 2024-12-06T05:24:53,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741859_1036 (size=20072) 2024-12-06T05:24:53,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741859_1036 (size=20072) 2024-12-06T05:24:53,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741859_1036 (size=20072) 2024-12-06T05:24:53,922 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/a/13ed9d6beb854f8f982472cee9b14b6d 2024-12-06T05:24:53,964 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/b/90dc341451194e0db920a4f5b77e2329 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733462688957/Put/seqid=0 2024-12-06T05:24:53,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741860_1037 (size=35835) 2024-12-06T05:24:53,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741860_1037 (size=35835) 2024-12-06T05:24:53,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741860_1037 (size=35835) 2024-12-06T05:24:53,984 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/b/90dc341451194e0db920a4f5b77e2329 2024-12-06T05:24:53,992 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/a/13ed9d6beb854f8f982472cee9b14b6d as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/a/13ed9d6beb854f8f982472cee9b14b6d 2024-12-06T05:24:53,999 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/a/13ed9d6beb854f8f982472cee9b14b6d, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-06T05:24:54,001 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/b/90dc341451194e0db920a4f5b77e2329 as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/b/90dc341451194e0db920a4f5b77e2329 2024-12-06T05:24:54,009 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/b/90dc341451194e0db920a4f5b77e2329, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-06T05:24:54,009 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 032405e2aa1e6e2fbc4fb6bf85f992b0 in 117ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-06T05:24:54,020 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-06T05:24:54,021 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 032405e2aa1e6e2fbc4fb6bf85f992b0 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-06T05:24:54,027 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/b/cba1456436ac4bd5ae42e980f053d5d6 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733462688986/Put/seqid=0 2024-12-06T05:24:54,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741861_1038 (size=35082) 2024-12-06T05:24:54,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741861_1038 (size=35082) 2024-12-06T05:24:54,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741861_1038 (size=35082) 2024-12-06T05:24:54,035 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/b/cba1456436ac4bd5ae42e980f053d5d6 2024-12-06T05:24:54,058 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/c/c4176bb1664b44c79231245275c223fb is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733462689036/Put/seqid=0 2024-12-06T05:24:54,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741862_1039 (size=20825) 2024-12-06T05:24:54,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741862_1039 (size=20825) 2024-12-06T05:24:54,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741862_1039 (size=20825) 2024-12-06T05:24:54,067 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/c/c4176bb1664b44c79231245275c223fb 2024-12-06T05:24:54,074 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/b/cba1456436ac4bd5ae42e980f053d5d6 as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/b/cba1456436ac4bd5ae42e980f053d5d6 2024-12-06T05:24:54,081 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/b/cba1456436ac4bd5ae42e980f053d5d6, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-06T05:24:54,083 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/c/c4176bb1664b44c79231245275c223fb as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/c/c4176bb1664b44c79231245275c223fb 2024-12-06T05:24:54,090 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/c/c4176bb1664b44c79231245275c223fb, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-06T05:24:54,090 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 032405e2aa1e6e2fbc4fb6bf85f992b0 in 70ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-06T05:24:54,108 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733462689109/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. 2024-12-06T05:24:54,112 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000003002 2024-12-06T05:24:54,112 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-06T05:24:54,113 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 032405e2aa1e6e2fbc4fb6bf85f992b0 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-06T05:24:54,122 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/c/36d659cb12834939931b99ee53dd7ede is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733462689050/Put/seqid=0 2024-12-06T05:24:54,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741863_1040 (size=50301) 2024-12-06T05:24:54,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741863_1040 (size=50301) 2024-12-06T05:24:54,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741863_1040 (size=50301) 2024-12-06T05:24:54,131 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/c/36d659cb12834939931b99ee53dd7ede 2024-12-06T05:24:54,139 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 36d659cb12834939931b99ee53dd7ede 2024-12-06T05:24:54,141 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/.tmp/c/36d659cb12834939931b99ee53dd7ede as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/c/36d659cb12834939931b99ee53dd7ede 2024-12-06T05:24:54,147 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 36d659cb12834939931b99ee53dd7ede 2024-12-06T05:24:54,147 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/c/36d659cb12834939931b99ee53dd7ede, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-06T05:24:54,148 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 032405e2aa1e6e2fbc4fb6bf85f992b0 in 35ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-06T05:24:54,149 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/0000000000000003002 2024-12-06T05:24:54,150 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:54,150 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:54,151 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-06T05:24:54,153 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 032405e2aa1e6e2fbc4fb6bf85f992b0 2024-12-06T05:24:54,155 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenIntoWAL/032405e2aa1e6e2fbc4fb6bf85f992b0/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-06T05:24:54,156 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 032405e2aa1e6e2fbc4fb6bf85f992b0; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65811836, jitterRate=-0.019327223300933838}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-06T05:24:54,157 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 032405e2aa1e6e2fbc4fb6bf85f992b0: Writing region info on filesystem at 1733462693354Initializing all the Stores at 1733462693355 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462693355Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462693356 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462693356Cleaning up temporary data from old regions at 1733462694151 (+795 ms)Region opened successfully at 1733462694156 (+5 ms) 2024-12-06T05:24:54,226 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 032405e2aa1e6e2fbc4fb6bf85f992b0, disabling compactions & flushes 2024-12-06T05:24:54,226 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. 2024-12-06T05:24:54,226 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. 2024-12-06T05:24:54,226 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. after waiting 0 ms 2024-12-06T05:24:54,226 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. 2024-12-06T05:24:54,238 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733462688771.032405e2aa1e6e2fbc4fb6bf85f992b0. 2024-12-06T05:24:54,238 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 032405e2aa1e6e2fbc4fb6bf85f992b0: Waiting for close lock at 1733462694225Disabling compacts and flushes for region at 1733462694225Disabling writes for close at 1733462694226 (+1 ms)Writing region close event to WAL at 1733462694237 (+11 ms)Closed at 1733462694238 (+1 ms) 2024-12-06T05:24:54,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741857_1034 (size=95) 2024-12-06T05:24:54,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741857_1034 (size=95) 2024-12-06T05:24:54,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741857_1034 (size=95) 2024-12-06T05:24:54,257 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:24:54,257 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733462693324) 2024-12-06T05:24:54,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=397 (was 385) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:40653 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1044892842_22 at /127.0.0.1:39102 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:45541 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1044892842_22 at /127.0.0.1:39108 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:45541 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1044892842_22 at /127.0.0.1:45256 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42941 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40653 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1044892842_22 at /127.0.0.1:37798 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:42941 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=265 (was 279), ProcessCount=11 (was 11), AvailableMemoryMB=7084 (was 7265) 2024-12-06T05:24:54,289 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=397, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=265, ProcessCount=11, AvailableMemoryMB=7082 2024-12-06T05:24:54,308 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:54,311 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:54,312 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:24:54,315 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91918665, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/hregion-91918665, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:54,327 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91918665/hregion-91918665.1733462694316, exclude list is [], retry=0 2024-12-06T05:24:54,331 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:54,331 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:54,331 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:54,334 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91918665/hregion-91918665.1733462694316 2024-12-06T05:24:54,334 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:24:54,334 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a9a7cc9423b5de2e48730492685616bf, NAME => 'test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/hbase 2024-12-06T05:24:54,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741865_1042 (size=43) 2024-12-06T05:24:54,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741865_1042 (size=43) 2024-12-06T05:24:54,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741865_1042 (size=43) 2024-12-06T05:24:54,351 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:54,353 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,355 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9a7cc9423b5de2e48730492685616bf columnFamilyName a 2024-12-06T05:24:54,355 DEBUG [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:54,356 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(327): Store=a9a7cc9423b5de2e48730492685616bf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:54,356 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,361 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9a7cc9423b5de2e48730492685616bf columnFamilyName b 2024-12-06T05:24:54,361 DEBUG [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:54,362 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(327): Store=a9a7cc9423b5de2e48730492685616bf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:54,362 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,364 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9a7cc9423b5de2e48730492685616bf columnFamilyName c 2024-12-06T05:24:54,364 DEBUG [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:54,365 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(327): Store=a9a7cc9423b5de2e48730492685616bf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:54,365 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,366 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,367 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,370 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,370 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,371 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:24:54,375 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,396 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:24:54,399 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a9a7cc9423b5de2e48730492685616bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60030040, jitterRate=-0.10548269748687744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:24:54,400 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a9a7cc9423b5de2e48730492685616bf: Writing region info on filesystem at 1733462694351Initializing all the Stores at 1733462694352 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462694352Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462694352Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462694352Cleaning up temporary data from old regions at 1733462694370 (+18 ms)Region opened successfully at 1733462694400 (+30 ms) 2024-12-06T05:24:54,401 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a9a7cc9423b5de2e48730492685616bf, disabling compactions & flushes 2024-12-06T05:24:54,401 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. 2024-12-06T05:24:54,401 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. 2024-12-06T05:24:54,401 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. after waiting 0 ms 2024-12-06T05:24:54,401 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. 2024-12-06T05:24:54,404 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. 2024-12-06T05:24:54,405 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a9a7cc9423b5de2e48730492685616bf: Waiting for close lock at 1733462694401Disabling compacts and flushes for region at 1733462694401Disabling writes for close at 1733462694401Writing region close event to WAL at 1733462694404 (+3 ms)Closed at 1733462694404 2024-12-06T05:24:54,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741864_1041 (size=95) 2024-12-06T05:24:54,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741864_1041 (size=95) 2024-12-06T05:24:54,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741864_1041 (size=95) 2024-12-06T05:24:54,422 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:24:54,422 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91918665:(num 1733462694316) 2024-12-06T05:24:54,422 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:24:54,425 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:54,439 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426, exclude list is [], retry=0 2024-12-06T05:24:54,443 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:54,444 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:54,444 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:54,446 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426 2024-12-06T05:24:54,446 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:24:54,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741866_1043 (size=263359) 2024-12-06T05:24:54,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741866_1043 (size=263359) 2024-12-06T05:24:54,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741866_1043 (size=263359) 2024-12-06T05:24:54,616 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426, size=257.2 K (263359bytes) 2024-12-06T05:24:54,616 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426 2024-12-06T05:24:54,617 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426 after 1ms 2024-12-06T05:24:54,619 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:54,621 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426 took 5ms 2024-12-06T05:24:54,625 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733462694426.temp 2024-12-06T05:24:54,627 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000000001-wal.1733462694426.temp 2024-12-06T05:24:54,679 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426 so closing down 2024-12-06T05:24:54,679 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:24:54,680 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:24:54,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741867_1044 (size=263359) 2024-12-06T05:24:54,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741867_1044 (size=263359) 2024-12-06T05:24:54,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741867_1044 (size=263359) 2024-12-06T05:24:54,685 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000000001-wal.1733462694426.temp (wrote 3000 edits, skipped 0 edits in 38 ms) 2024-12-06T05:24:54,688 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000000001-wal.1733462694426.temp to hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003000 2024-12-06T05:24:54,689 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-12-06T05:24:54,689 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426, journal: Splitting hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426, size=257.2 K (263359bytes) at 1733462694616Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000000001-wal.1733462694426.temp at 1733462694627 (+11 ms)Split 1024 edits, skipped 0 edits. at 1733462694644 (+17 ms)Split 2048 edits, skipped 0 edits. at 1733462694661 (+17 ms)Finishing writing output for hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426 so closing down at 1733462694679 (+18 ms)3 split writer threads finished at 1733462694680 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000000001-wal.1733462694426.temp (wrote 3000 edits, skipped 0 edits in 38 ms) at 1733462694686 (+6 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000000001-wal.1733462694426.temp to hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003000 at 1733462694688 (+2 ms)Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1733462694689 (+1 ms) 2024-12-06T05:24:54,694 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694426 to hdfs://localhost:45541/hbase/oldWALs/wal.1733462694426 2024-12-06T05:24:54,697 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003000 2024-12-06T05:24:54,698 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:24:54,701 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:54,720 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702, exclude list is [], retry=0 2024-12-06T05:24:54,724 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:54,725 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:54,725 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:54,733 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702 2024-12-06T05:24:54,736 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:24:54,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741868_1045 (size=263486) 2024-12-06T05:24:54,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741868_1045 (size=263486) 2024-12-06T05:24:54,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741868_1045 (size=263486) 2024-12-06T05:24:54,887 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702, size=257.3 K (263486bytes) 2024-12-06T05:24:54,887 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702 2024-12-06T05:24:54,888 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702 after 1ms 2024-12-06T05:24:54,891 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:54,892 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702 took 5ms 2024-12-06T05:24:54,897 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733462694702.temp 2024-12-06T05:24:54,898 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003001-wal.1733462694702.temp 2024-12-06T05:24:54,939 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-06T05:24:54,940 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-06T05:24:54,942 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T05:24:54,942 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-06T05:24:54,943 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-06T05:24:54,943 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-06T05:24:54,944 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-06T05:24:54,944 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-06T05:24:54,949 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702 so closing down 2024-12-06T05:24:54,949 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:24:54,955 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:24:54,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741869_1046 (size=263486) 2024-12-06T05:24:54,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741869_1046 (size=263486) 2024-12-06T05:24:54,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741869_1046 (size=263486) 2024-12-06T05:24:54,961 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003001-wal.1733462694702.temp (wrote 3000 edits, skipped 0 edits in 36 ms) 2024-12-06T05:24:54,963 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003001-wal.1733462694702.temp to hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000006000 2024-12-06T05:24:54,963 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-12-06T05:24:54,963 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702, journal: Splitting hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702, size=257.3 K (263486bytes) at 1733462694887Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003001-wal.1733462694702.temp at 1733462694898 (+11 ms)Split 1024 edits, skipped 0 edits. at 1733462694913 (+15 ms)Split 2048 edits, skipped 0 edits. at 1733462694932 (+19 ms)Finishing writing output for hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702 so closing down at 1733462694949 (+17 ms)3 split writer threads finished at 1733462694955 (+6 ms)Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003001-wal.1733462694702.temp (wrote 3000 edits, skipped 0 edits in 36 ms) at 1733462694961 (+6 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003001-wal.1733462694702.temp to hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000006000 at 1733462694963 (+2 ms)Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1733462694963 2024-12-06T05:24:54,966 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694702 to hdfs://localhost:45541/hbase/oldWALs/wal.1733462694702 2024-12-06T05:24:54,967 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000006000 2024-12-06T05:24:54,967 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:24:54,970 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/test2727-manual,16010,1733462694307, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:54,984 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694970, exclude list is [], retry=0 2024-12-06T05:24:54,987 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:54,988 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:54,988 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:54,991 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733462694307/wal.1733462694970 2024-12-06T05:24:54,992 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:24:54,992 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a9a7cc9423b5de2e48730492685616bf, NAME => 'test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:24:54,992 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:54,992 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,992 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,995 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,996 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9a7cc9423b5de2e48730492685616bf columnFamilyName a 2024-12-06T05:24:54,996 DEBUG [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:54,997 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(327): Store=a9a7cc9423b5de2e48730492685616bf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:54,997 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:54,999 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9a7cc9423b5de2e48730492685616bf columnFamilyName b 2024-12-06T05:24:54,999 DEBUG [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:55,000 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(327): Store=a9a7cc9423b5de2e48730492685616bf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:55,000 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:55,001 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9a7cc9423b5de2e48730492685616bf columnFamilyName c 2024-12-06T05:24:55,001 DEBUG [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:55,002 INFO [StoreOpener-a9a7cc9423b5de2e48730492685616bf-1 {}] regionserver.HStore(327): Store=a9a7cc9423b5de2e48730492685616bf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:55,002 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:55,003 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:55,005 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:55,006 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003000 2024-12-06T05:24:55,009 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:55,059 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003000 2024-12-06T05:24:55,060 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000006000 2024-12-06T05:24:55,066 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:55,112 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000006000 2024-12-06T05:24:55,112 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a9a7cc9423b5de2e48730492685616bf 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-06T05:24:55,131 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/.tmp/a/a546ae3a2a304369b907719d89e2f15e is 41, key is test2727/a:100/1733462694740/Put/seqid=0 2024-12-06T05:24:55,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741871_1048 (size=84227) 2024-12-06T05:24:55,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741871_1048 (size=84227) 2024-12-06T05:24:55,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741871_1048 (size=84227) 2024-12-06T05:24:55,139 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/.tmp/a/a546ae3a2a304369b907719d89e2f15e 2024-12-06T05:24:55,172 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/.tmp/b/87e42789787242ddbec56b188b690133 is 41, key is test2727/b:100/1733462694784/Put/seqid=0 2024-12-06T05:24:55,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741872_1049 (size=84609) 2024-12-06T05:24:55,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741872_1049 (size=84609) 2024-12-06T05:24:55,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741872_1049 (size=84609) 2024-12-06T05:24:55,181 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/.tmp/b/87e42789787242ddbec56b188b690133 2024-12-06T05:24:55,209 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/.tmp/c/c8116195b5b945a2a7db079c060583f7 is 41, key is test2727/c:100/1733462694825/Put/seqid=0 2024-12-06T05:24:55,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741873_1050 (size=84609) 2024-12-06T05:24:55,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741873_1050 (size=84609) 2024-12-06T05:24:55,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741873_1050 (size=84609) 2024-12-06T05:24:55,218 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/.tmp/c/c8116195b5b945a2a7db079c060583f7 2024-12-06T05:24:55,226 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/.tmp/a/a546ae3a2a304369b907719d89e2f15e as hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/a/a546ae3a2a304369b907719d89e2f15e 2024-12-06T05:24:55,233 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/a/a546ae3a2a304369b907719d89e2f15e, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-06T05:24:55,235 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/.tmp/b/87e42789787242ddbec56b188b690133 as hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/b/87e42789787242ddbec56b188b690133 2024-12-06T05:24:55,242 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/b/87e42789787242ddbec56b188b690133, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-06T05:24:55,244 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/.tmp/c/c8116195b5b945a2a7db079c060583f7 as hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/c/c8116195b5b945a2a7db079c060583f7 2024-12-06T05:24:55,252 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/c/c8116195b5b945a2a7db079c060583f7, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-06T05:24:55,252 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for a9a7cc9423b5de2e48730492685616bf in 140ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-06T05:24:55,254 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000003000 2024-12-06T05:24:55,254 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/0000000000000006000 2024-12-06T05:24:55,256 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:55,256 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:55,256 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:24:55,258 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a9a7cc9423b5de2e48730492685616bf 2024-12-06T05:24:55,261 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/test2727/a9a7cc9423b5de2e48730492685616bf/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-06T05:24:55,262 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a9a7cc9423b5de2e48730492685616bf; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59310895, jitterRate=-0.11619879305362701}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:24:55,264 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a9a7cc9423b5de2e48730492685616bf: Writing region info on filesystem at 1733462694992Initializing all the Stores at 1733462694994 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462694994Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462694995 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462694995Obtaining lock to block concurrent updates at 1733462695112 (+117 ms)Preparing flush snapshotting stores in a9a7cc9423b5de2e48730492685616bf at 1733462695112Finished memstore snapshotting test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733462695112Flushing stores of test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. at 1733462695112Flushing a9a7cc9423b5de2e48730492685616bf/a: creating writer at 1733462695112Flushing a9a7cc9423b5de2e48730492685616bf/a: appending metadata at 1733462695130 (+18 ms)Flushing a9a7cc9423b5de2e48730492685616bf/a: closing flushed file at 1733462695131 (+1 ms)Flushing a9a7cc9423b5de2e48730492685616bf/b: creating writer at 1733462695147 (+16 ms)Flushing a9a7cc9423b5de2e48730492685616bf/b: appending metadata at 1733462695170 (+23 ms)Flushing a9a7cc9423b5de2e48730492685616bf/b: closing flushed file at 1733462695170Flushing a9a7cc9423b5de2e48730492685616bf/c: creating writer at 1733462695189 (+19 ms)Flushing a9a7cc9423b5de2e48730492685616bf/c: appending metadata at 1733462695208 (+19 ms)Flushing a9a7cc9423b5de2e48730492685616bf/c: closing flushed file at 1733462695208Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@690fea7e: reopening flushed file at 1733462695225 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54675698: reopening flushed file at 1733462695234 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@101df2fd: reopening flushed file at 1733462695242 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for a9a7cc9423b5de2e48730492685616bf in 140ms, sequenceid=6000, compaction requested=false; wal=null at 1733462695252 (+10 ms)Cleaning up temporary data from old regions at 1733462695256 (+4 ms)Region opened successfully at 1733462695264 (+8 ms) 2024-12-06T05:24:55,265 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-06T05:24:55,265 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a9a7cc9423b5de2e48730492685616bf, disabling compactions & flushes 2024-12-06T05:24:55,265 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. 2024-12-06T05:24:55,265 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. 2024-12-06T05:24:55,265 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. after waiting 0 ms 2024-12-06T05:24:55,265 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. 2024-12-06T05:24:55,267 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733462694309.a9a7cc9423b5de2e48730492685616bf. 2024-12-06T05:24:55,267 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a9a7cc9423b5de2e48730492685616bf: Waiting for close lock at 1733462695265Disabling compacts and flushes for region at 1733462695265Disabling writes for close at 1733462695265Writing region close event to WAL at 1733462695267 (+2 ms)Closed at 1733462695267 2024-12-06T05:24:55,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741870_1047 (size=95) 2024-12-06T05:24:55,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741870_1047 (size=95) 2024-12-06T05:24:55,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741870_1047 (size=95) 2024-12-06T05:24:55,273 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:24:55,274 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733462694970) 2024-12-06T05:24:55,285 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=401 (was 397) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:45338 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:39234 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:37882 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=265 (was 265), ProcessCount=11 (was 11), AvailableMemoryMB=6886 (was 7082) 2024-12-06T05:24:55,295 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=401, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=265, ProcessCount=11, AvailableMemoryMB=6885 2024-12-06T05:24:55,313 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:55,319 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:55,319 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733462695319 2024-12-06T05:24:55,325 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319 2024-12-06T05:24:55,327 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:24:55,328 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 939efa6be64740fa0b0c6c7d3f5af140, NAME => 'testSequentialEditLogSeqNum,,1733462695314.939efa6be64740fa0b0c6c7d3f5af140.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:24:55,328 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733462695314.939efa6be64740fa0b0c6c7d3f5af140.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:55,328 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 939efa6be64740fa0b0c6c7d3f5af140 2024-12-06T05:24:55,328 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 939efa6be64740fa0b0c6c7d3f5af140 2024-12-06T05:24:55,329 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140 doesn't exist for region: 939efa6be64740fa0b0c6c7d3f5af140 on table testSequentialEditLogSeqNum 2024-12-06T05:24:55,330 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 939efa6be64740fa0b0c6c7d3f5af140 on table testSequentialEditLogSeqNum 2024-12-06T05:24:55,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741875_1052 (size=62) 2024-12-06T05:24:55,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741875_1052 (size=62) 2024-12-06T05:24:55,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741875_1052 (size=62) 2024-12-06T05:24:55,339 INFO [StoreOpener-939efa6be64740fa0b0c6c7d3f5af140-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 939efa6be64740fa0b0c6c7d3f5af140 2024-12-06T05:24:55,341 INFO [StoreOpener-939efa6be64740fa0b0c6c7d3f5af140-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 939efa6be64740fa0b0c6c7d3f5af140 columnFamilyName a 2024-12-06T05:24:55,341 DEBUG [StoreOpener-939efa6be64740fa0b0c6c7d3f5af140-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:55,342 INFO [StoreOpener-939efa6be64740fa0b0c6c7d3f5af140-1 {}] regionserver.HStore(327): Store=939efa6be64740fa0b0c6c7d3f5af140/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:55,342 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 939efa6be64740fa0b0c6c7d3f5af140 2024-12-06T05:24:55,343 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140 2024-12-06T05:24:55,343 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140 2024-12-06T05:24:55,343 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 939efa6be64740fa0b0c6c7d3f5af140 2024-12-06T05:24:55,343 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 939efa6be64740fa0b0c6c7d3f5af140 2024-12-06T05:24:55,345 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 939efa6be64740fa0b0c6c7d3f5af140 2024-12-06T05:24:55,348 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:24:55,349 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 939efa6be64740fa0b0c6c7d3f5af140; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64852811, jitterRate=-0.033617809414863586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:24:55,350 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 939efa6be64740fa0b0c6c7d3f5af140: Writing region info on filesystem at 1733462695329Initializing all the Stores at 1733462695339 (+10 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462695339Cleaning up temporary data from old regions at 1733462695343 (+4 ms)Region opened successfully at 1733462695350 (+7 ms) 2024-12-06T05:24:55,365 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 939efa6be64740fa0b0c6c7d3f5af140 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-06T05:24:55,386 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/.tmp/a/4b71a989e3124d37b1d94297e91bca3e is 81, key is testSequentialEditLogSeqNum/a:x0/1733462695350/Put/seqid=0 2024-12-06T05:24:55,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741876_1053 (size=5833) 2024-12-06T05:24:55,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741876_1053 (size=5833) 2024-12-06T05:24:55,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741876_1053 (size=5833) 2024-12-06T05:24:55,394 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/.tmp/a/4b71a989e3124d37b1d94297e91bca3e 2024-12-06T05:24:55,401 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/.tmp/a/4b71a989e3124d37b1d94297e91bca3e as hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/a/4b71a989e3124d37b1d94297e91bca3e 2024-12-06T05:24:55,408 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/a/4b71a989e3124d37b1d94297e91bca3e, entries=10, sequenceid=13, filesize=5.7 K 2024-12-06T05:24:55,409 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 939efa6be64740fa0b0c6c7d3f5af140 in 44ms, sequenceid=13, compaction requested=false 2024-12-06T05:24:55,409 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 939efa6be64740fa0b0c6c7d3f5af140: 2024-12-06T05:24:55,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-06T05:24:55,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-06T05:24:55,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-06T05:24:55,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-06T05:24:55,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-06T05:24:55,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741874_1051 (size=1844) 2024-12-06T05:24:55,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741874_1051 (size=1844) 2024-12-06T05:24:55,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741874_1051 (size=1844) 2024-12-06T05:24:55,432 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319, size=1.8 K (1844bytes) 2024-12-06T05:24:55,433 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319 2024-12-06T05:24:55,433 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319 after 0ms 2024-12-06T05:24:55,436 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:55,436 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319 took 4ms 2024-12-06T05:24:55,440 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319 so closing down 2024-12-06T05:24:55,440 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:24:55,441 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733462695319.temp 2024-12-06T05:24:55,442 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/recovered.edits/0000000000000000003-wal.1733462695319.temp 2024-12-06T05:24:55,443 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:24:55,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741877_1054 (size=1477) 2024-12-06T05:24:55,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741877_1054 (size=1477) 2024-12-06T05:24:55,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741877_1054 (size=1477) 2024-12-06T05:24:55,456 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/recovered.edits/0000000000000000003-wal.1733462695319.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-06T05:24:55,458 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/recovered.edits/0000000000000000003-wal.1733462695319.temp to hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/recovered.edits/0000000000000000020 2024-12-06T05:24:55,459 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 22 ms; skipped=2; WAL=hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319, size=1.8 K, length=1844, corrupted=false, cancelled=false 2024-12-06T05:24:55,459 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319, journal: Splitting hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319, size=1.8 K (1844bytes) at 1733462695432Finishing writing output for hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319 so closing down at 1733462695440 (+8 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/recovered.edits/0000000000000000003-wal.1733462695319.temp at 1733462695442 (+2 ms)3 split writer threads finished at 1733462695443 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/recovered.edits/0000000000000000003-wal.1733462695319.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733462695456 (+13 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/recovered.edits/0000000000000000003-wal.1733462695319.temp to hdfs://localhost:45541/hbase/data/default/testSequentialEditLogSeqNum/939efa6be64740fa0b0c6c7d3f5af140/recovered.edits/0000000000000000020 at 1733462695458 (+2 ms)Processed 17 edits across 1 Regions in 22 ms; skipped=2; WAL=hdfs://localhost:45541/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733462695312/wal.1733462695319, size=1.8 K, length=1844, corrupted=false, cancelled=false at 1733462695459 (+1 ms) 2024-12-06T05:24:55,474 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=406 (was 401) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:45338 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:39234 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:37882 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=265 (was 265), ProcessCount=11 (was 11), AvailableMemoryMB=6861 (was 6885) 2024-12-06T05:24:55,486 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=265, ProcessCount=11, AvailableMemoryMB=6859 2024-12-06T05:24:55,501 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:55,503 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:55,533 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:24:55,536 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-12316275, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/hregion-12316275, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:55,549 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-12316275/hregion-12316275.1733462695537, exclude list is [], retry=0 2024-12-06T05:24:55,553 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:55,553 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:55,554 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:55,560 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-12316275/hregion-12316275.1733462695537 2024-12-06T05:24:55,561 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:24:55,561 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 38eef531c438a3ebe3492af62fde87a9, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/hbase 2024-12-06T05:24:55,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741879_1056 (size=70) 2024-12-06T05:24:55,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741879_1056 (size=70) 2024-12-06T05:24:55,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741879_1056 (size=70) 2024-12-06T05:24:55,597 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:55,599 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,602 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38eef531c438a3ebe3492af62fde87a9 columnFamilyName a 2024-12-06T05:24:55,602 DEBUG [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:55,604 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(327): Store=38eef531c438a3ebe3492af62fde87a9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:55,605 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,607 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38eef531c438a3ebe3492af62fde87a9 columnFamilyName b 2024-12-06T05:24:55,607 DEBUG [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:55,607 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(327): Store=38eef531c438a3ebe3492af62fde87a9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:55,608 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,609 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38eef531c438a3ebe3492af62fde87a9 columnFamilyName c 2024-12-06T05:24:55,610 DEBUG [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:55,611 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(327): Store=38eef531c438a3ebe3492af62fde87a9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:55,612 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,613 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,613 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,615 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,615 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,616 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:24:55,618 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,622 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:24:55,623 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 38eef531c438a3ebe3492af62fde87a9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73462535, jitterRate=0.09467707574367523}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:24:55,624 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 38eef531c438a3ebe3492af62fde87a9: Writing region info on filesystem at 1733462695597Initializing all the Stores at 1733462695598 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462695598Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462695598Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462695599 (+1 ms)Cleaning up temporary data from old regions at 1733462695615 (+16 ms)Region opened successfully at 1733462695624 (+9 ms) 2024-12-06T05:24:55,624 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 38eef531c438a3ebe3492af62fde87a9, disabling compactions & flushes 2024-12-06T05:24:55,624 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. 2024-12-06T05:24:55,624 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. 2024-12-06T05:24:55,624 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. after waiting 0 ms 2024-12-06T05:24:55,625 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. 2024-12-06T05:24:55,625 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. 2024-12-06T05:24:55,625 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 38eef531c438a3ebe3492af62fde87a9: Waiting for close lock at 1733462695624Disabling compacts and flushes for region at 1733462695624Disabling writes for close at 1733462695625 (+1 ms)Writing region close event to WAL at 1733462695625Closed at 1733462695625 2024-12-06T05:24:55,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741878_1055 (size=95) 2024-12-06T05:24:55,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741878_1055 (size=95) 2024-12-06T05:24:55,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741878_1055 (size=95) 2024-12-06T05:24:55,635 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:24:55,635 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-12316275:(num 1733462695537) 2024-12-06T05:24:55,635 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:24:55,638 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:55,659 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639, exclude list is [], retry=0 2024-12-06T05:24:55,662 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:55,662 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:55,663 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:55,665 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 2024-12-06T05:24:55,666 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:24:55,666 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 38eef531c438a3ebe3492af62fde87a9, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:24:55,666 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:55,667 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,667 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,672 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,674 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38eef531c438a3ebe3492af62fde87a9 columnFamilyName a 2024-12-06T05:24:55,674 DEBUG [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:55,675 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(327): Store=38eef531c438a3ebe3492af62fde87a9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:55,675 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,676 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38eef531c438a3ebe3492af62fde87a9 columnFamilyName b 2024-12-06T05:24:55,676 DEBUG [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:55,677 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(327): Store=38eef531c438a3ebe3492af62fde87a9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:55,677 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,678 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38eef531c438a3ebe3492af62fde87a9 columnFamilyName c 2024-12-06T05:24:55,678 DEBUG [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:55,678 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(327): Store=38eef531c438a3ebe3492af62fde87a9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:55,679 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,680 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,681 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,682 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,682 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,683 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:24:55,685 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:55,686 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 38eef531c438a3ebe3492af62fde87a9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65145897, jitterRate=-0.029250487685203552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:24:55,687 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 38eef531c438a3ebe3492af62fde87a9: Writing region info on filesystem at 1733462695667Initializing all the Stores at 1733462695668 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462695668Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462695672 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462695672Cleaning up temporary data from old regions at 1733462695682 (+10 ms)Region opened successfully at 1733462695687 (+5 ms) 2024-12-06T05:24:55,692 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733462695691/Put/seqid=0 2024-12-06T05:24:55,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741881_1058 (size=4826) 2024-12-06T05:24:55,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741881_1058 (size=4826) 2024-12-06T05:24:55,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741881_1058 (size=4826) 2024-12-06T05:24:55,704 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45541/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 38eef531c438a3ebe3492af62fde87a9/a 2024-12-06T05:24:55,713 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-06T05:24:55,713 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-06T05:24:55,714 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 38eef531c438a3ebe3492af62fde87a9: 2024-12-06T05:24:55,716 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/a/82fa1d9468b84e1ca6f5209c35b8ec80_SeqId_3_ 2024-12-06T05:24:55,717 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45541/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 38eef531c438a3ebe3492af62fde87a9/a as hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/a/82fa1d9468b84e1ca6f5209c35b8ec80_SeqId_3_ - updating store file list. 2024-12-06T05:24:55,724 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 82fa1d9468b84e1ca6f5209c35b8ec80_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-06T05:24:55,724 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/a/82fa1d9468b84e1ca6f5209c35b8ec80_SeqId_3_ into 38eef531c438a3ebe3492af62fde87a9/a 2024-12-06T05:24:55,724 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45541/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 38eef531c438a3ebe3492af62fde87a9/a (new location: hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/a/82fa1d9468b84e1ca6f5209c35b8ec80_SeqId_3_) 2024-12-06T05:24:55,763 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639, size=0 (0bytes) 2024-12-06T05:24:55,763 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 might be still open, length is 0 2024-12-06T05:24:55,763 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 2024-12-06T05:24:55,763 WARN [IPC Server handler 4 on default port 45541 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-06T05:24:55,764 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 after 1ms 2024-12-06T05:24:58,726 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:45424 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43097:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45424 dst: /127.0.0.1:43097 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43097 remote=/127.0.0.1:45424]. Total timeout mills is 60000, 57005 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:24:58,727 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:37976 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45599:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37976 dst: /127.0.0.1:45599 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:24:58,727 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:39338 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:35849:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39338 dst: /127.0.0.1:35849 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:24:58,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741880_1059 (size=473) 2024-12-06T05:24:58,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741880_1059 (size=473) 2024-12-06T05:24:58,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741880_1059 (size=473) 2024-12-06T05:24:59,765 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 after 4002ms 2024-12-06T05:24:59,772 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:59,772 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 took 4010ms 2024-12-06T05:24:59,776 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639; continuing. 2024-12-06T05:24:59,776 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 so closing down 2024-12-06T05:24:59,776 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:24:59,778 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733462695639.temp 2024-12-06T05:24:59,780 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005-wal.1733462695639.temp 2024-12-06T05:24:59,780 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:24:59,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741882_1060 (size=259) 2024-12-06T05:24:59,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741882_1060 (size=259) 2024-12-06T05:24:59,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741882_1060 (size=259) 2024-12-06T05:24:59,788 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005-wal.1733462695639.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-06T05:24:59,790 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005-wal.1733462695639.temp to hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005 2024-12-06T05:24:59,790 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639, size=0, length=0, corrupted=false, cancelled=false 2024-12-06T05:24:59,791 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639, journal: Splitting hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639, size=0 (0bytes) at 1733462695763Finishing writing output for hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 so closing down at 1733462699776 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005-wal.1733462695639.temp at 1733462699780 (+4 ms)3 split writer threads finished at 1733462699780Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005-wal.1733462695639.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733462699788 (+8 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005-wal.1733462695639.temp to hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005 at 1733462699790 (+2 ms)Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639, size=0, length=0, corrupted=false, cancelled=false at 1733462699790 2024-12-06T05:24:59,793 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 to hdfs://localhost:45541/hbase/oldWALs/wal.1733462695639 2024-12-06T05:24:59,794 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005 2024-12-06T05:24:59,795 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:24:59,797 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:24:59,809 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462699797, exclude list is [], retry=0 2024-12-06T05:24:59,811 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:24:59,812 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:24:59,812 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:24:59,814 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462699797 2024-12-06T05:24:59,814 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:24:59,815 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 38eef531c438a3ebe3492af62fde87a9, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:24:59,815 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:59,815 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,815 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,816 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,817 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38eef531c438a3ebe3492af62fde87a9 columnFamilyName a 2024-12-06T05:24:59,817 DEBUG [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:59,823 DEBUG [StoreFileOpener-38eef531c438a3ebe3492af62fde87a9-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 82fa1d9468b84e1ca6f5209c35b8ec80_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-06T05:24:59,824 DEBUG [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/a/82fa1d9468b84e1ca6f5209c35b8ec80_SeqId_3_ 2024-12-06T05:24:59,824 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(327): Store=38eef531c438a3ebe3492af62fde87a9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:59,824 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,825 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38eef531c438a3ebe3492af62fde87a9 columnFamilyName b 2024-12-06T05:24:59,825 DEBUG [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:59,826 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(327): Store=38eef531c438a3ebe3492af62fde87a9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:59,826 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,827 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 38eef531c438a3ebe3492af62fde87a9 columnFamilyName c 2024-12-06T05:24:59,827 DEBUG [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:59,828 INFO [StoreOpener-38eef531c438a3ebe3492af62fde87a9-1 {}] regionserver.HStore(327): Store=38eef531c438a3ebe3492af62fde87a9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:24:59,828 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,828 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,830 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,830 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005 2024-12-06T05:24:59,832 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:24:59,833 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005 2024-12-06T05:24:59,833 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 38eef531c438a3ebe3492af62fde87a9 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-06T05:24:59,846 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/.tmp/a/8a45ff6f3d434d00ab965c69263a8884 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733462695729/Put/seqid=0 2024-12-06T05:24:59,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741884_1062 (size=5149) 2024-12-06T05:24:59,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741884_1062 (size=5149) 2024-12-06T05:24:59,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741884_1062 (size=5149) 2024-12-06T05:24:59,855 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/.tmp/a/8a45ff6f3d434d00ab965c69263a8884 2024-12-06T05:24:59,860 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/.tmp/a/8a45ff6f3d434d00ab965c69263a8884 as hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/a/8a45ff6f3d434d00ab965c69263a8884 2024-12-06T05:24:59,866 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/a/8a45ff6f3d434d00ab965c69263a8884, entries=1, sequenceid=5, filesize=5.0 K 2024-12-06T05:24:59,867 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 38eef531c438a3ebe3492af62fde87a9 in 34ms, sequenceid=5, compaction requested=false; wal=null 2024-12-06T05:24:59,867 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/0000000000000000005 2024-12-06T05:24:59,869 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,869 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,870 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:24:59,872 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 38eef531c438a3ebe3492af62fde87a9 2024-12-06T05:24:59,874 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/38eef531c438a3ebe3492af62fde87a9/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-06T05:24:59,875 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 38eef531c438a3ebe3492af62fde87a9; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69899742, jitterRate=0.04158732295036316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:24:59,876 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 38eef531c438a3ebe3492af62fde87a9: Writing region info on filesystem at 1733462699815Initializing all the Stores at 1733462699816 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462699816Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462699816Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462699816Obtaining lock to block concurrent updates at 1733462699833 (+17 ms)Preparing flush snapshotting stores in 38eef531c438a3ebe3492af62fde87a9 at 1733462699833Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733462699833Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. at 1733462699833Flushing 38eef531c438a3ebe3492af62fde87a9/a: creating writer at 1733462699834 (+1 ms)Flushing 38eef531c438a3ebe3492af62fde87a9/a: appending metadata at 1733462699846 (+12 ms)Flushing 38eef531c438a3ebe3492af62fde87a9/a: closing flushed file at 1733462699846Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47fc2c29: reopening flushed file at 1733462699859 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 38eef531c438a3ebe3492af62fde87a9 in 34ms, sequenceid=5, compaction requested=false; wal=null at 1733462699867 (+8 ms)Cleaning up temporary data from old regions at 1733462699869 (+2 ms)Region opened successfully at 1733462699876 (+7 ms) 2024-12-06T05:24:59,880 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 38eef531c438a3ebe3492af62fde87a9, disabling compactions & flushes 2024-12-06T05:24:59,880 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. 2024-12-06T05:24:59,880 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. 2024-12-06T05:24:59,880 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. after waiting 0 ms 2024-12-06T05:24:59,880 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. 2024-12-06T05:24:59,881 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733462695501.38eef531c438a3ebe3492af62fde87a9. 2024-12-06T05:24:59,881 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 38eef531c438a3ebe3492af62fde87a9: Waiting for close lock at 1733462699880Disabling compacts and flushes for region at 1733462699880Disabling writes for close at 1733462699880Writing region close event to WAL at 1733462699881 (+1 ms)Closed at 1733462699881 2024-12-06T05:24:59,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741883_1061 (size=95) 2024-12-06T05:24:59,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741883_1061 (size=95) 2024-12-06T05:24:59,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741883_1061 (size=95) 2024-12-06T05:24:59,886 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:24:59,886 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733462699797) 2024-12-06T05:24:59,899 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=413 (was 408) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-280737450_22 at /127.0.0.1:39382 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:46255 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:45541 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-280737450_22 at /127.0.0.1:38030 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46255 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:45541 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-280737450_22 at /127.0.0.1:45428 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=252 (was 265), ProcessCount=11 (was 11), AvailableMemoryMB=7487 (was 6859) - AvailableMemoryMB LEAK? - 2024-12-06T05:24:59,900 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-06T05:24:59,915 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=413, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=252, ProcessCount=11, AvailableMemoryMB=7486 2024-12-06T05:24:59,915 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-06T05:24:59,932 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:24:59,936 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-06T05:24:59,940 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ec46afd49254,43357,1733462684569 2024-12-06T05:24:59,942 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61339409 2024-12-06T05:24:59,942 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-06T05:24:59,944 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-06T05:24:59,948 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-06T05:24:59,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-06T05:24:59,955 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-06T05:24:59,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-06T05:24:59,958 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:24:59,959 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-06T05:24:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:24:59,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741885_1063 (size=694) 2024-12-06T05:24:59,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741885_1063 (size=694) 2024-12-06T05:24:59,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741885_1063 (size=694) 2024-12-06T05:24:59,971 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 55114b43c8aa4ca85a42aa15503261cc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727 2024-12-06T05:24:59,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741886_1064 (size=77) 2024-12-06T05:24:59,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741886_1064 (size=77) 2024-12-06T05:24:59,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741886_1064 (size=77) 2024-12-06T05:24:59,983 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:24:59,983 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 55114b43c8aa4ca85a42aa15503261cc, disabling compactions & flushes 2024-12-06T05:24:59,983 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:24:59,983 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:24:59,983 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. after waiting 0 ms 2024-12-06T05:24:59,983 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:24:59,983 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:24:59,983 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 55114b43c8aa4ca85a42aa15503261cc: Waiting for close lock at 1733462699983Disabling compacts and flushes for region at 1733462699983Disabling writes for close at 1733462699983Writing region close event to WAL at 1733462699983Closed at 1733462699983 2024-12-06T05:24:59,985 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-06T05:24:59,992 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733462699985"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733462699985"}]},"ts":"1733462699985"} 2024-12-06T05:24:59,996 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-06T05:24:59,998 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-06T05:25:00,000 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733462699998"}]},"ts":"1733462699998"} 2024-12-06T05:25:00,004 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-06T05:25:00,005 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {ec46afd49254=0} racks are {/default-rack=0} 2024-12-06T05:25:00,006 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:25:00,006 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:25:00,006 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-06T05:25:00,006 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:25:00,006 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:25:00,006 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-06T05:25:00,006 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:25:00,006 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:25:00,007 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-06T05:25:00,007 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:25:00,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, ASSIGN}] 2024-12-06T05:25:00,011 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, ASSIGN 2024-12-06T05:25:00,012 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, ASSIGN; state=OFFLINE, location=ec46afd49254,39229,1733462685524; forceNewPlan=false, retain=false 2024-12-06T05:25:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:25:00,168 INFO [ec46afd49254:43357 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T05:25:00,170 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=OPENING, regionLocation=ec46afd49254,39229,1733462685524 2024-12-06T05:25:00,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, ASSIGN because future has completed 2024-12-06T05:25:00,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,39229,1733462685524}] 2024-12-06T05:25:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:25:00,329 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:25:00,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37589, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:25:00,335 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,336 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 55114b43c8aa4ca85a42aa15503261cc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:00,336 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,336 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:00,336 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,336 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,338 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,339 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf1 2024-12-06T05:25:00,339 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:00,340 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:00,340 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,341 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf2 2024-12-06T05:25:00,341 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:00,342 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:00,342 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,343 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,343 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,344 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,344 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,345 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-06T05:25:00,347 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,349 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:25:00,350 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 55114b43c8aa4ca85a42aa15503261cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61965104, jitterRate=-0.07664799690246582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-06T05:25:00,350 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,351 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 55114b43c8aa4ca85a42aa15503261cc: Running coprocessor pre-open hook at 1733462700336Writing region info on filesystem at 1733462700336Initializing all the Stores at 1733462700337 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462700337Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462700338 (+1 ms)Cleaning up temporary data from old regions at 1733462700344 (+6 ms)Running coprocessor post-open hooks at 1733462700350 (+6 ms)Region opened successfully at 1733462700351 (+1 ms) 2024-12-06T05:25:00,353 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., pid=6, masterSystemTime=1733462700329 2024-12-06T05:25:00,356 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,356 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,357 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=OPEN, openSeqNum=2, regionLocation=ec46afd49254,39229,1733462685524 2024-12-06T05:25:00,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,39229,1733462685524 because future has completed 2024-12-06T05:25:00,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-06T05:25:00,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,39229,1733462685524 in 186 msec 2024-12-06T05:25:00,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-06T05:25:00,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, ASSIGN in 357 msec 2024-12-06T05:25:00,370 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-06T05:25:00,370 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733462700370"}]},"ts":"1733462700370"} 2024-12-06T05:25:00,372 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-06T05:25:00,373 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-06T05:25:00,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 424 msec 2024-12-06T05:25:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-06T05:25:00,589 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-06T05:25:00,589 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-06T05:25:00,590 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T05:25:00,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-06T05:25:00,597 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-06T05:25:00,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-06T05:25:00,608 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=2] 2024-12-06T05:25:00,609 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-06T05:25:00,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-06T05:25:00,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=55114b43c8aa4ca85a42aa15503261cc, source=ec46afd49254,39229,1733462685524, destination=ec46afd49254,43053,1733462685378, warming up region on ec46afd49254,43053,1733462685378 2024-12-06T05:25:00,627 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=55114b43c8aa4ca85a42aa15503261cc, source=ec46afd49254,39229,1733462685524, destination=ec46afd49254,43053,1733462685378, running balancer 2024-12-06T05:25:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE 2024-12-06T05:25:00,629 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE 2024-12-06T05:25:00,631 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=CLOSING, regionLocation=ec46afd49254,39229,1733462685524 2024-12-06T05:25:00,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] regionserver.HRegion(7855): Warmup {ENCODED => 55114b43c8aa4ca85a42aa15503261cc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:00,633 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,634 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf1 2024-12-06T05:25:00,635 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:00,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE because future has completed 2024-12-06T05:25:00,635 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:00,635 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T05:25:00,635 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,39229,1733462685524}] 2024-12-06T05:25:00,636 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf2 2024-12-06T05:25:00,636 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:00,637 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] regionserver.HRegion(1722): Closing 55114b43c8aa4ca85a42aa15503261cc, disabling compactions & flushes 2024-12-06T05:25:00,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. after waiting 0 ms 2024-12-06T05:25:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] regionserver.HRegion(1676): Region close journal for 55114b43c8aa4ca85a42aa15503261cc: Waiting for close lock at 1733462700637Disabling compacts and flushes for region at 1733462700637Disabling writes for close at 1733462700637Writing region close event to WAL at 1733462700638 (+1 ms)Closed at 1733462700638 2024-12-06T05:25:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-06T05:25:00,794 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,794 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T05:25:00,795 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 55114b43c8aa4ca85a42aa15503261cc, disabling compactions & flushes 2024-12-06T05:25:00,795 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,795 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,795 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. after waiting 0 ms 2024-12-06T05:25:00,795 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,795 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 55114b43c8aa4ca85a42aa15503261cc 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-06T05:25:00,815 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf1/629b78fc31554143a46fdc82bc42afc6 is 35, key is r1/cf1:q/1733462700611/Put/seqid=0 2024-12-06T05:25:00,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741887_1065 (size=4783) 2024-12-06T05:25:00,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741887_1065 (size=4783) 2024-12-06T05:25:00,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741887_1065 (size=4783) 2024-12-06T05:25:00,822 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf1/629b78fc31554143a46fdc82bc42afc6 2024-12-06T05:25:00,830 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf1/629b78fc31554143a46fdc82bc42afc6 as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/629b78fc31554143a46fdc82bc42afc6 2024-12-06T05:25:00,837 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/629b78fc31554143a46fdc82bc42afc6, entries=1, sequenceid=5, filesize=4.7 K 2024-12-06T05:25:00,838 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 55114b43c8aa4ca85a42aa15503261cc in 43ms, sequenceid=5, compaction requested=false 2024-12-06T05:25:00,838 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-06T05:25:00,844 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-06T05:25:00,846 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:00,846 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 55114b43c8aa4ca85a42aa15503261cc: Waiting for close lock at 1733462700795Running coprocessor pre-close hooks at 1733462700795Disabling compacts and flushes for region at 1733462700795Disabling writes for close at 1733462700795Obtaining lock to block concurrent updates at 1733462700796 (+1 ms)Preparing flush snapshotting stores in 55114b43c8aa4ca85a42aa15503261cc at 1733462700796Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733462700796Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. at 1733462700797 (+1 ms)Flushing 55114b43c8aa4ca85a42aa15503261cc/cf1: creating writer at 1733462700797Flushing 55114b43c8aa4ca85a42aa15503261cc/cf1: appending metadata at 1733462700815 (+18 ms)Flushing 55114b43c8aa4ca85a42aa15503261cc/cf1: closing flushed file at 1733462700815Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f6ec425: reopening flushed file at 1733462700829 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 55114b43c8aa4ca85a42aa15503261cc in 43ms, sequenceid=5, compaction requested=false at 1733462700838 (+9 ms)Writing region close event to WAL at 1733462700840 (+2 ms)Running coprocessor post-close hooks at 1733462700844 (+4 ms)Closed at 1733462700846 (+2 ms) 2024-12-06T05:25:00,846 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 55114b43c8aa4ca85a42aa15503261cc move to ec46afd49254,43053,1733462685378 record at close sequenceid=5 2024-12-06T05:25:00,850 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:00,851 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=CLOSED 2024-12-06T05:25:00,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,39229,1733462685524 because future has completed 2024-12-06T05:25:00,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-06T05:25:00,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,39229,1733462685524 in 219 msec 2024-12-06T05:25:00,859 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE; state=CLOSED, location=ec46afd49254,43053,1733462685378; forceNewPlan=false, retain=false 2024-12-06T05:25:01,009 INFO [ec46afd49254:43357 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T05:25:01,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=OPENING, regionLocation=ec46afd49254,43053,1733462685378 2024-12-06T05:25:01,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE because future has completed 2024-12-06T05:25:01,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,43053,1733462685378}] 2024-12-06T05:25:01,174 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,175 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 55114b43c8aa4ca85a42aa15503261cc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:01,175 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,175 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:01,175 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,175 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,177 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,178 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf1 2024-12-06T05:25:01,178 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:01,186 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/629b78fc31554143a46fdc82bc42afc6 2024-12-06T05:25:01,186 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:01,186 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,187 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf2 2024-12-06T05:25:01,187 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:01,188 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:01,188 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,189 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,190 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,191 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,191 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,192 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-06T05:25:01,194 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,195 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 55114b43c8aa4ca85a42aa15503261cc; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70307722, jitterRate=0.047666698694229126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-06T05:25:01,195 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,195 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 55114b43c8aa4ca85a42aa15503261cc: Running coprocessor pre-open hook at 1733462701175Writing region info on filesystem at 1733462701175Initializing all the Stores at 1733462701176 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462701176Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462701177 (+1 ms)Cleaning up temporary data from old regions at 1733462701191 (+14 ms)Running coprocessor post-open hooks at 1733462701195 (+4 ms)Region opened successfully at 1733462701195 2024-12-06T05:25:01,197 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., pid=9, masterSystemTime=1733462701170 2024-12-06T05:25:01,199 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,199 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,200 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=OPEN, openSeqNum=9, regionLocation=ec46afd49254,43053,1733462685378 2024-12-06T05:25:01,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,43053,1733462685378 because future has completed 2024-12-06T05:25:01,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-06T05:25:01,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,43053,1733462685378 in 188 msec 2024-12-06T05:25:01,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE in 580 msec 2024-12-06T05:25:01,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:25:01,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:25:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:47688 deadline: 1733462761240, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec46afd49254 port=43053 startCode=1733462685378. As of locationSeqNum=5. 2024-12-06T05:25:01,246 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec46afd49254 port=43053 startCode=1733462685378. As of locationSeqNum=5. 2024-12-06T05:25:01,247 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec46afd49254 port=43053 startCode=1733462685378. As of locationSeqNum=5. 2024-12-06T05:25:01,247 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,43053,1733462685378, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec46afd49254 port=43053 startCode=1733462685378. As of locationSeqNum=5. 2024-12-06T05:25:01,367 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 55114b43c8aa4ca85a42aa15503261cc 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-06T05:25:01,383 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf1/fe58afaa59644b2c89a65ceedda1ef83 is 29, key is r1/cf1:/1733462701360/DeleteFamily/seqid=0 2024-12-06T05:25:01,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741888_1066 (size=4906) 2024-12-06T05:25:01,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741888_1066 (size=4906) 2024-12-06T05:25:01,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741888_1066 (size=4906) 2024-12-06T05:25:01,390 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf1/fe58afaa59644b2c89a65ceedda1ef83 2024-12-06T05:25:01,397 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fe58afaa59644b2c89a65ceedda1ef83 2024-12-06T05:25:01,411 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf2/1f78ea293c0649998dfa4096f6e3a4bf is 29, key is r1/cf2:/1733462701360/DeleteFamily/seqid=0 2024-12-06T05:25:01,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741889_1067 (size=4906) 2024-12-06T05:25:01,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741889_1067 (size=4906) 2024-12-06T05:25:01,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741889_1067 (size=4906) 2024-12-06T05:25:01,418 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf2/1f78ea293c0649998dfa4096f6e3a4bf 2024-12-06T05:25:01,424 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1f78ea293c0649998dfa4096f6e3a4bf 2024-12-06T05:25:01,425 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf1/fe58afaa59644b2c89a65ceedda1ef83 as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/fe58afaa59644b2c89a65ceedda1ef83 2024-12-06T05:25:01,432 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fe58afaa59644b2c89a65ceedda1ef83 2024-12-06T05:25:01,432 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/fe58afaa59644b2c89a65ceedda1ef83, entries=1, sequenceid=12, filesize=4.8 K 2024-12-06T05:25:01,433 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf2/1f78ea293c0649998dfa4096f6e3a4bf as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/1f78ea293c0649998dfa4096f6e3a4bf 2024-12-06T05:25:01,440 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1f78ea293c0649998dfa4096f6e3a4bf 2024-12-06T05:25:01,440 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/1f78ea293c0649998dfa4096f6e3a4bf, entries=1, sequenceid=12, filesize=4.8 K 2024-12-06T05:25:01,441 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 55114b43c8aa4ca85a42aa15503261cc in 74ms, sequenceid=12, compaction requested=false 2024-12-06T05:25:01,442 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 55114b43c8aa4ca85a42aa15503261cc: 2024-12-06T05:25:01,444 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-06T05:25:01,445 DEBUG [Time-limited test {}] regionserver.HStore(1541): 55114b43c8aa4ca85a42aa15503261cc/cf1 is initiating major compaction (all files) 2024-12-06T05:25:01,445 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:25:01,445 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:25:01,446 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 55114b43c8aa4ca85a42aa15503261cc/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,446 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/629b78fc31554143a46fdc82bc42afc6, hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/fe58afaa59644b2c89a65ceedda1ef83] into tmpdir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp, totalSize=9.5 K 2024-12-06T05:25:01,447 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 629b78fc31554143a46fdc82bc42afc6, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733462700611 2024-12-06T05:25:01,448 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fe58afaa59644b2c89a65ceedda1ef83, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-06T05:25:01,460 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 55114b43c8aa4ca85a42aa15503261cc#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T05:25:01,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741890_1068 (size=4626) 2024-12-06T05:25:01,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741890_1068 (size=4626) 2024-12-06T05:25:01,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741890_1068 (size=4626) 2024-12-06T05:25:01,478 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf1/6a76148265fc46dfab437f896552343e as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/6a76148265fc46dfab437f896552343e 2024-12-06T05:25:01,494 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 55114b43c8aa4ca85a42aa15503261cc/cf1 of 55114b43c8aa4ca85a42aa15503261cc into 6a76148265fc46dfab437f896552343e(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T05:25:01,494 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 55114b43c8aa4ca85a42aa15503261cc: 2024-12-06T05:25:01,495 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-06T05:25:01,495 DEBUG [Time-limited test {}] regionserver.HStore(1541): 55114b43c8aa4ca85a42aa15503261cc/cf2 is initiating major compaction (all files) 2024-12-06T05:25:01,495 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-06T05:25:01,495 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-06T05:25:01,495 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 55114b43c8aa4ca85a42aa15503261cc/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,495 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/1f78ea293c0649998dfa4096f6e3a4bf] into tmpdir=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp, totalSize=4.8 K 2024-12-06T05:25:01,496 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1f78ea293c0649998dfa4096f6e3a4bf, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-06T05:25:01,503 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 55114b43c8aa4ca85a42aa15503261cc#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-06T05:25:01,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741891_1069 (size=4592) 2024-12-06T05:25:01,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741891_1069 (size=4592) 2024-12-06T05:25:01,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741891_1069 (size=4592) 2024-12-06T05:25:01,520 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/.tmp/cf2/a92d34a6e1fc4fb1907611ae533b0f82 as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/a92d34a6e1fc4fb1907611ae533b0f82 2024-12-06T05:25:01,528 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 55114b43c8aa4ca85a42aa15503261cc/cf2 of 55114b43c8aa4ca85a42aa15503261cc into a92d34a6e1fc4fb1907611ae533b0f82(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T05:25:01,528 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 55114b43c8aa4ca85a42aa15503261cc: 2024-12-06T05:25:01,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=55114b43c8aa4ca85a42aa15503261cc, source=ec46afd49254,43053,1733462685378, destination=ec46afd49254,39229,1733462685524, warming up region on ec46afd49254,39229,1733462685524 2024-12-06T05:25:01,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=55114b43c8aa4ca85a42aa15503261cc, source=ec46afd49254,43053,1733462685378, destination=ec46afd49254,39229,1733462685524, running balancer 2024-12-06T05:25:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE 2024-12-06T05:25:01,534 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE 2024-12-06T05:25:01,536 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=CLOSING, regionLocation=ec46afd49254,43053,1733462685378 2024-12-06T05:25:01,536 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] regionserver.HRegion(7855): Warmup {ENCODED => 55114b43c8aa4ca85a42aa15503261cc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:01,537 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,539 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf1 2024-12-06T05:25:01,539 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:01,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE because future has completed 2024-12-06T05:25:01,539 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-06T05:25:01,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,43053,1733462685378}] 2024-12-06T05:25:01,547 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/629b78fc31554143a46fdc82bc42afc6 2024-12-06T05:25:01,552 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/6a76148265fc46dfab437f896552343e 2024-12-06T05:25:01,557 INFO [StoreFileOpener-55114b43c8aa4ca85a42aa15503261cc-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fe58afaa59644b2c89a65ceedda1ef83 2024-12-06T05:25:01,557 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/fe58afaa59644b2c89a65ceedda1ef83 2024-12-06T05:25:01,557 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:01,557 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,558 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf2 2024-12-06T05:25:01,559 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:01,565 INFO [StoreFileOpener-55114b43c8aa4ca85a42aa15503261cc-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1f78ea293c0649998dfa4096f6e3a4bf 2024-12-06T05:25:01,565 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/1f78ea293c0649998dfa4096f6e3a4bf 2024-12-06T05:25:01,570 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/a92d34a6e1fc4fb1907611ae533b0f82 2024-12-06T05:25:01,570 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] regionserver.HRegion(1722): Closing 55114b43c8aa4ca85a42aa15503261cc, disabling compactions & flushes 2024-12-06T05:25:01,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. after waiting 0 ms 2024-12-06T05:25:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39229 {}] regionserver.HRegion(1676): Region close journal for 55114b43c8aa4ca85a42aa15503261cc: Waiting for close lock at 1733462701570Disabling compacts and flushes for region at 1733462701570Disabling writes for close at 1733462701571 (+1 ms)Writing region close event to WAL at 1733462701572 (+1 ms)Closed at 1733462701572 2024-12-06T05:25:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-06T05:25:01,694 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,694 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-06T05:25:01,694 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 55114b43c8aa4ca85a42aa15503261cc, disabling compactions & flushes 2024-12-06T05:25:01,694 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,694 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,694 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. after waiting 0 ms 2024-12-06T05:25:01,694 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,695 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/629b78fc31554143a46fdc82bc42afc6, hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/fe58afaa59644b2c89a65ceedda1ef83] to archive 2024-12-06T05:25:01,698 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T05:25:01,701 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/629b78fc31554143a46fdc82bc42afc6 to hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/629b78fc31554143a46fdc82bc42afc6 2024-12-06T05:25:01,703 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/fe58afaa59644b2c89a65ceedda1ef83 to hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/fe58afaa59644b2c89a65ceedda1ef83 2024-12-06T05:25:01,716 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/1f78ea293c0649998dfa4096f6e3a4bf] to archive 2024-12-06T05:25:01,717 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T05:25:01,720 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/1f78ea293c0649998dfa4096f6e3a4bf to hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/1f78ea293c0649998dfa4096f6e3a4bf 2024-12-06T05:25:01,725 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-06T05:25:01,726 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:01,726 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 55114b43c8aa4ca85a42aa15503261cc: Waiting for close lock at 1733462701694Running coprocessor pre-close hooks at 1733462701694Disabling compacts and flushes for region at 1733462701694Disabling writes for close at 1733462701694Writing region close event to WAL at 1733462701721 (+27 ms)Running coprocessor post-close hooks at 1733462701726 (+5 ms)Closed at 1733462701726 2024-12-06T05:25:01,726 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 55114b43c8aa4ca85a42aa15503261cc move to ec46afd49254,39229,1733462685524 record at close sequenceid=12 2024-12-06T05:25:01,729 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:01,729 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=CLOSED 2024-12-06T05:25:01,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,43053,1733462685378 because future has completed 2024-12-06T05:25:01,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-06T05:25:01,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,43053,1733462685378 in 193 msec 2024-12-06T05:25:01,738 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE; state=CLOSED, location=ec46afd49254,39229,1733462685524; forceNewPlan=false, retain=false 2024-12-06T05:25:01,888 INFO [ec46afd49254:43357 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-06T05:25:01,889 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=OPENING, regionLocation=ec46afd49254,39229,1733462685524 2024-12-06T05:25:01,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE because future has completed 2024-12-06T05:25:01,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,39229,1733462685524}] 2024-12-06T05:25:02,057 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:02,057 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 55114b43c8aa4ca85a42aa15503261cc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:02,058 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,058 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:02,058 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,058 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,060 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,062 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf1 2024-12-06T05:25:02,062 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:02,071 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/6a76148265fc46dfab437f896552343e 2024-12-06T05:25:02,071 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:02,071 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,072 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf2 2024-12-06T05:25:02,072 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:02,080 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/a92d34a6e1fc4fb1907611ae533b0f82 2024-12-06T05:25:02,081 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:02,081 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,082 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,083 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,084 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,084 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,085 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-06T05:25:02,086 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,087 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 55114b43c8aa4ca85a42aa15503261cc; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70588226, jitterRate=0.05184653401374817}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-06T05:25:02,087 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,088 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 55114b43c8aa4ca85a42aa15503261cc: Running coprocessor pre-open hook at 1733462702059Writing region info on filesystem at 1733462702059Initializing all the Stores at 1733462702060 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462702060Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462702060Cleaning up temporary data from old regions at 1733462702084 (+24 ms)Running coprocessor post-open hooks at 1733462702087 (+3 ms)Region opened successfully at 1733462702088 (+1 ms) 2024-12-06T05:25:02,089 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., pid=12, masterSystemTime=1733462702049 2024-12-06T05:25:02,091 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:02,091 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:02,091 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=OPEN, openSeqNum=18, regionLocation=ec46afd49254,39229,1733462685524 2024-12-06T05:25:02,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,39229,1733462685524 because future has completed 2024-12-06T05:25:02,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-06T05:25:02,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,39229,1733462685524 in 199 msec 2024-12-06T05:25:02,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, REOPEN/MOVE in 564 msec 2024-12-06T05:25:02,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-06T05:25:02,139 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-06T05:25:02,140 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server ec46afd49254,39229,1733462685524: testing ***** 2024-12-06T05:25:02,141 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-06T05:25:02,143 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-06T05:25:02,144 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-06T05:25:02,147 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-06T05:25:02,148 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-06T05:25:02,156 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 320908800 }, "NonHeapMemoryUsage": { "committed": 170786816, "init": 7667712, "max": -1, "used": 168265024 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "ec46afd49254", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 12, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 6, "ProcessCallTime_75th_percentile": 9, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 12, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 9, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "ec46afd49254", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:63857", "tag.serverName": "ec46afd49254,42449,1733462685474", "tag.clusterId": "c8507a94-304b-414d-b222-aad5346dd76a", "tag.Context": "regionserver", "tag.Hostname": "ec46afd49254", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733462685474, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-06T05:25:02,160 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43357 {}] master.MasterRpcServices(700): ec46afd49254,39229,1733462685524 reported a fatal error: ***** ABORTING region server ec46afd49254,39229,1733462685524: testing ***** 2024-12-06T05:25:02,162 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,39229,1733462685524' ***** 2024-12-06T05:25:02,162 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-06T05:25:02,162 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:25:02,163 INFO [RS:2;ec46afd49254:39229 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-06T05:25:02,163 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:25:02,163 INFO [RS:2;ec46afd49254:39229 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-06T05:25:02,163 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(3091): Received CLOSE for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,163 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(956): aborting server ec46afd49254,39229,1733462685524 2024-12-06T05:25:02,163 INFO [RS:2;ec46afd49254:39229 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:25:02,164 INFO [RS:2;ec46afd49254:39229 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ec46afd49254:39229. 2024-12-06T05:25:02,164 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 55114b43c8aa4ca85a42aa15503261cc, disabling compactions & flushes 2024-12-06T05:25:02,164 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:02,164 DEBUG [RS:2;ec46afd49254:39229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:25:02,164 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:02,164 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. after waiting 0 ms 2024-12-06T05:25:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43053 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:60024 deadline: 1733462762164, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec46afd49254 port=39229 startCode=1733462685524. As of locationSeqNum=12. 2024-12-06T05:25:02,164 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:02,164 DEBUG [RS:2;ec46afd49254:39229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:25:02,165 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,43053,1733462685378, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,43053,1733462685378, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec46afd49254 port=39229 startCode=1733462685524. As of locationSeqNum=12. 2024-12-06T05:25:02,165 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,43053,1733462685378, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec46afd49254 port=39229 startCode=1733462685524. As of locationSeqNum=12. 2024-12-06T05:25:02,165 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,43053,1733462685378, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=ec46afd49254 port=39229 startCode=1733462685524. As of locationSeqNum=12. 2024-12-06T05:25:02,167 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-06T05:25:02,167 DEBUG [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(1325): Online Regions={55114b43c8aa4ca85a42aa15503261cc=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.} 2024-12-06T05:25:02,167 DEBUG [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(1351): Waiting on 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:02,168 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:02,168 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 55114b43c8aa4ca85a42aa15503261cc: Waiting for close lock at 1733462702163Running coprocessor pre-close hooks at 1733462702164 (+1 ms)Disabling compacts and flushes for region at 1733462702164Disabling writes for close at 1733462702164Writing region close event to WAL at 1733462702168 (+4 ms)Running coprocessor post-close hooks at 1733462702168Closed at 1733462702168 2024-12-06T05:25:02,168 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:02,227 INFO [regionserver/ec46afd49254:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:25:02,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ec46afd49254,39229,1733462685524 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:02,279 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ec46afd49254,39229,1733462685524 aborting 2024-12-06T05:25:02,279 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server ec46afd49254,39229,1733462685524 aborting 2024-12-06T05:25:02,279 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=12 from cache 2024-12-06T05:25:02,368 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,39229,1733462685524; all regions closed. 2024-12-06T05:25:02,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741833_1009 (size=1403) 2024-12-06T05:25:02,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741833_1009 (size=1403) 2024-12-06T05:25:02,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741833_1009 (size=1403) 2024-12-06T05:25:02,373 DEBUG [RS:2;ec46afd49254:39229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:25:02,373 INFO [RS:2;ec46afd49254:39229 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:25:02,373 INFO [RS:2;ec46afd49254:39229 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:25:02,373 INFO [RS:2;ec46afd49254:39229 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-06T05:25:02,374 INFO [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:25:02,374 INFO [RS:2;ec46afd49254:39229 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:25:02,374 INFO [RS:2;ec46afd49254:39229 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:25:02,374 INFO [RS:2;ec46afd49254:39229 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:25:02,374 INFO [RS:2;ec46afd49254:39229 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:25:02,374 INFO [RS:2;ec46afd49254:39229 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39229 2024-12-06T05:25:02,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,39229,1733462685524 2024-12-06T05:25:02,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:25:02,440 INFO [RS:2;ec46afd49254:39229 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:25:02,443 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,39229,1733462685524] 2024-12-06T05:25:02,461 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,39229,1733462685524 already deleted, retry=false 2024-12-06T05:25:02,461 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ec46afd49254,39229,1733462685524 on ec46afd49254,43357,1733462684569 2024-12-06T05:25:02,468 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ec46afd49254,39229,1733462685524, splitWal=true, meta=false 2024-12-06T05:25:02,470 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for ec46afd49254,39229,1733462685524 (carryingMeta=false) ec46afd49254,39229,1733462685524/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1365dc86[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-06T05:25:02,471 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ec46afd49254,39229,1733462685524, splitWal=true, meta=false 2024-12-06T05:25:02,472 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): ec46afd49254,39229,1733462685524 had 1 regions 2024-12-06T05:25:02,474 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ec46afd49254,39229,1733462685524, splitWal=true, meta=false, isMeta: false 2024-12-06T05:25:02,475 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting 2024-12-06T05:25:02,476 INFO [PEWorker-2 {}] master.SplitWALManager(105): ec46afd49254,39229,1733462685524 WAL count=1, meta=false 2024-12-06T05:25:02,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure ec46afd49254%2C39229%2C1733462685524.1733462687186}] 2024-12-06T05:25:02,482 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=ec46afd49254,43053,1733462685378 2024-12-06T05:25:02,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure ec46afd49254%2C39229%2C1733462685524.1733462687186, worker=ec46afd49254,43053,1733462685378}] 2024-12-06T05:25:02,492 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=18] 2024-12-06T05:25:02,493 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ec46afd49254:39229 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ec46afd49254/172.17.0.3:39229 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:02,494 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=18, error=java.net.ConnectException: Call to address=ec46afd49254:39229 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ec46afd49254/172.17.0.3:39229 2024-12-06T05:25:02,494 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-06T05:25:02,494 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=18 from cache 2024-12-06T05:25:02,494 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address ec46afd49254:39229 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: ec46afd49254/172.17.0.3:39229 2024-12-06T05:25:02,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:25:02,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39229-0x101a9196c580003, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:25:02,552 INFO [RS:2;ec46afd49254:39229 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:25:02,552 INFO [RS:2;ec46afd49254:39229 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,39229,1733462685524; zookeeper connection closed. 2024-12-06T05:25:02,552 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a95a8bf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a95a8bf 2024-12-06T05:25:02,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43053 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-06T05:25:02,666 INFO [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186, size=1.4 K (1403bytes) 2024-12-06T05:25:02,666 INFO [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186 2024-12-06T05:25:02,667 INFO [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186 after 1ms 2024-12-06T05:25:02,669 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:02,669 INFO [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186 took 3ms 2024-12-06T05:25:02,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 55114b43c8aa4ca85a42aa15503261cc: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-06T05:25:02,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186 so closing down 2024-12-06T05:25:02,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:25:02,678 INFO [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:25:02,678 INFO [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186, size=1.4 K, length=1403, corrupted=false, cancelled=false 2024-12-06T05:25:02,678 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186, journal: Splitting hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186, size=1.4 K (1403bytes) at 1733462702666Finishing writing output for hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186 so closing down at 1733462702677 (+11 ms)3 split writer threads finished at 1733462702678 (+1 ms)Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186, size=1.4 K, length=1403, corrupted=false, cancelled=false at 1733462702678 2024-12-06T05:25:02,678 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186 2024-12-06T05:25:02,679 DEBUG [RS_LOG_REPLAY_OPS-regionserver/ec46afd49254:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-06T05:25:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43357 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-06T05:25:02,685 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting/ec46afd49254%2C39229%2C1733462685524.1733462687186 to hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/oldWALs 2024-12-06T05:25:02,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-06T05:25:02,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure ec46afd49254%2C39229%2C1733462685524.1733462687186, worker=ec46afd49254,43053,1733462685378 in 201 msec 2024-12-06T05:25:02,689 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=ec46afd49254,43053,1733462685378 2024-12-06T05:25:02,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-06T05:25:02,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure ec46afd49254%2C39229%2C1733462685524.1733462687186, worker=ec46afd49254,43053,1733462685378 in 212 msec 2024-12-06T05:25:02,693 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting dir is empty, no logs to split. 2024-12-06T05:25:02,693 INFO [PEWorker-2 {}] master.SplitWALManager(105): ec46afd49254,39229,1733462685524 WAL count=0, meta=false 2024-12-06T05:25:02,694 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if ec46afd49254,39229,1733462685524 WAL splitting is done? wals=0, meta=false 2024-12-06T05:25:02,696 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for ec46afd49254,39229,1733462685524 failed, ignore...File hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/WALs/ec46afd49254,39229,1733462685524-splitting does not exist. 2024-12-06T05:25:02,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, ASSIGN}] 2024-12-06T05:25:02,700 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, ASSIGN 2024-12-06T05:25:02,701 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-06T05:25:02,800 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=18] 2024-12-06T05:25:02,801 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to ec46afd49254:39229 this server is in the failed servers list 2024-12-06T05:25:02,801 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=ec46afd49254:39229 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ec46afd49254:39229 2024-12-06T05:25:02,801 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: ec46afd49254:39229 2024-12-06T05:25:02,801 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,39229,1733462685524, seqNum=18 from cache 2024-12-06T05:25:02,851 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(204): Hosts are {ec46afd49254=0} racks are {/default-rack=0} 2024-12-06T05:25:02,852 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-06T05:25:02,852 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-06T05:25:02,852 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-06T05:25:02,852 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-06T05:25:02,852 INFO [ec46afd49254:43357 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-06T05:25:02,852 INFO [ec46afd49254:43357 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-06T05:25:02,852 DEBUG [ec46afd49254:43357 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-06T05:25:02,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=OPENING, regionLocation=ec46afd49254,43053,1733462685378 2024-12-06T05:25:02,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, ASSIGN because future has completed 2024-12-06T05:25:02,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,43053,1733462685378}] 2024-12-06T05:25:03,018 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:03,018 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 55114b43c8aa4ca85a42aa15503261cc, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:03,019 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,019 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:03,019 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,019 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,022 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,024 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf1 2024-12-06T05:25:03,024 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,036 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf1/6a76148265fc46dfab437f896552343e 2024-12-06T05:25:03,037 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,037 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,039 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 55114b43c8aa4ca85a42aa15503261cc columnFamilyName cf2 2024-12-06T05:25:03,039 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,050 DEBUG [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/cf2/a92d34a6e1fc4fb1907611ae533b0f82 2024-12-06T05:25:03,050 INFO [StoreOpener-55114b43c8aa4ca85a42aa15503261cc-1 {}] regionserver.HStore(327): Store=55114b43c8aa4ca85a42aa15503261cc/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,050 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,052 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,054 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,055 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,055 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,056 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-06T05:25:03,057 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,058 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 55114b43c8aa4ca85a42aa15503261cc; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68792322, jitterRate=0.025085479021072388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-06T05:25:03,058 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:03,059 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 55114b43c8aa4ca85a42aa15503261cc: Running coprocessor pre-open hook at 1733462703020Writing region info on filesystem at 1733462703020Initializing all the Stores at 1733462703021 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703021Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703022 (+1 ms)Cleaning up temporary data from old regions at 1733462703055 (+33 ms)Running coprocessor post-open hooks at 1733462703058 (+3 ms)Region opened successfully at 1733462703059 (+1 ms) 2024-12-06T05:25:03,059 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., pid=17, masterSystemTime=1733462703010 2024-12-06T05:25:03,062 DEBUG [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:03,062 INFO [RS_OPEN_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:03,062 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=55114b43c8aa4ca85a42aa15503261cc, regionState=OPEN, openSeqNum=18, regionLocation=ec46afd49254,43053,1733462685378 2024-12-06T05:25:03,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,43053,1733462685378 because future has completed 2024-12-06T05:25:03,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-06T05:25:03,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 55114b43c8aa4ca85a42aa15503261cc, server=ec46afd49254,43053,1733462685378 in 209 msec 2024-12-06T05:25:03,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-06T05:25:03,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=55114b43c8aa4ca85a42aa15503261cc, ASSIGN in 370 msec 2024-12-06T05:25:03,070 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server ec46afd49254,39229,1733462685524 after splitting done 2024-12-06T05:25:03,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure ec46afd49254,39229,1733462685524, splitWal=true, meta=false in 607 msec 2024-12-06T05:25:03,312 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc., hostname=ec46afd49254,43053,1733462685378, seqNum=18] 2024-12-06T05:25:03,328 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=404 (was 413), OpenFileDescriptor=1045 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 252) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7456 (was 7486) 2024-12-06T05:25:03,329 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1045 is superior to 1024 2024-12-06T05:25:03,339 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=404, OpenFileDescriptor=1045, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=7456 2024-12-06T05:25:03,339 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1045 is superior to 1024 2024-12-06T05:25:03,351 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:25:03,353 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:25:03,353 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:25:03,355 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-02662825, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/hregion-02662825, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:03,368 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-02662825/hregion-02662825.1733462703355, exclude list is [], retry=0 2024-12-06T05:25:03,371 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:03,371 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:03,371 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:03,373 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-02662825/hregion-02662825.1733462703355 2024-12-06T05:25:03,374 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:25:03,374 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ea3e4b760d1e140747a9cd737237c04c, NAME => 'testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/hbase 2024-12-06T05:25:03,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741893_1071 (size=67) 2024-12-06T05:25:03,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741893_1071 (size=67) 2024-12-06T05:25:03,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741893_1071 (size=67) 2024-12-06T05:25:03,385 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:03,386 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,388 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea3e4b760d1e140747a9cd737237c04c columnFamilyName a 2024-12-06T05:25:03,388 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,388 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(327): Store=ea3e4b760d1e140747a9cd737237c04c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,389 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,390 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea3e4b760d1e140747a9cd737237c04c columnFamilyName b 2024-12-06T05:25:03,390 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,391 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(327): Store=ea3e4b760d1e140747a9cd737237c04c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,391 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,392 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea3e4b760d1e140747a9cd737237c04c columnFamilyName c 2024-12-06T05:25:03,392 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,393 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(327): Store=ea3e4b760d1e140747a9cd737237c04c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,393 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,394 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,394 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,394 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,395 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:03,396 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,398 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:25:03,398 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea3e4b760d1e140747a9cd737237c04c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62702290, jitterRate=-0.06566306948661804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:03,399 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea3e4b760d1e140747a9cd737237c04c: Writing region info on filesystem at 1733462703385Initializing all the Stores at 1733462703386 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703386Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703386Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703386Cleaning up temporary data from old regions at 1733462703394 (+8 ms)Region opened successfully at 1733462703399 (+5 ms) 2024-12-06T05:25:03,399 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea3e4b760d1e140747a9cd737237c04c, disabling compactions & flushes 2024-12-06T05:25:03,399 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. 2024-12-06T05:25:03,399 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. 2024-12-06T05:25:03,399 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. after waiting 0 ms 2024-12-06T05:25:03,399 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. 2024-12-06T05:25:03,400 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. 2024-12-06T05:25:03,400 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea3e4b760d1e140747a9cd737237c04c: Waiting for close lock at 1733462703399Disabling compacts and flushes for region at 1733462703399Disabling writes for close at 1733462703399Writing region close event to WAL at 1733462703400 (+1 ms)Closed at 1733462703400 2024-12-06T05:25:03,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741892_1070 (size=95) 2024-12-06T05:25:03,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741892_1070 (size=95) 2024-12-06T05:25:03,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741892_1070 (size=95) 2024-12-06T05:25:03,404 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:25:03,404 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-02662825:(num 1733462703355) 2024-12-06T05:25:03,405 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:03,406 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:03,420 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407, exclude list is [], retry=0 2024-12-06T05:25:03,422 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:03,422 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:03,423 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:03,424 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407 2024-12-06T05:25:03,425 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:25:03,425 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea3e4b760d1e140747a9cd737237c04c, NAME => 'testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:03,425 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:03,425 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,425 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,426 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,427 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea3e4b760d1e140747a9cd737237c04c columnFamilyName a 2024-12-06T05:25:03,427 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,428 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(327): Store=ea3e4b760d1e140747a9cd737237c04c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,428 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,428 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea3e4b760d1e140747a9cd737237c04c columnFamilyName b 2024-12-06T05:25:03,428 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,429 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(327): Store=ea3e4b760d1e140747a9cd737237c04c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,429 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,430 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea3e4b760d1e140747a9cd737237c04c columnFamilyName c 2024-12-06T05:25:03,430 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,430 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(327): Store=ea3e4b760d1e140747a9cd737237c04c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,430 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,431 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,433 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,434 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,434 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,435 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:03,436 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,437 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea3e4b760d1e140747a9cd737237c04c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68822795, jitterRate=0.025539562106132507}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:03,437 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea3e4b760d1e140747a9cd737237c04c: Writing region info on filesystem at 1733462703425Initializing all the Stores at 1733462703426 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703426Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703426Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703426Cleaning up temporary data from old regions at 1733462703434 (+8 ms)Region opened successfully at 1733462703437 (+3 ms) 2024-12-06T05:25:03,462 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea3e4b760d1e140747a9cd737237c04c 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-06T05:25:03,477 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/a/4db6d72b5a774554b1e9fd48bcfb9ffb is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733462703437/Put/seqid=0 2024-12-06T05:25:03,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741895_1073 (size=5958) 2024-12-06T05:25:03,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741895_1073 (size=5958) 2024-12-06T05:25:03,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741895_1073 (size=5958) 2024-12-06T05:25:03,484 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/a/4db6d72b5a774554b1e9fd48bcfb9ffb 2024-12-06T05:25:03,502 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/b/74ddeca3a27d461aa8cf5479ec41615a is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733462703447/Put/seqid=0 2024-12-06T05:25:03,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741896_1074 (size=5958) 2024-12-06T05:25:03,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741896_1074 (size=5958) 2024-12-06T05:25:03,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741896_1074 (size=5958) 2024-12-06T05:25:03,509 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/b/74ddeca3a27d461aa8cf5479ec41615a 2024-12-06T05:25:03,526 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/c/78fae4b3efc44e779da75150df447d36 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733462703454/Put/seqid=0 2024-12-06T05:25:03,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741897_1075 (size=5958) 2024-12-06T05:25:03,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741897_1075 (size=5958) 2024-12-06T05:25:03,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741897_1075 (size=5958) 2024-12-06T05:25:03,532 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/c/78fae4b3efc44e779da75150df447d36 2024-12-06T05:25:03,538 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/a/4db6d72b5a774554b1e9fd48bcfb9ffb as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/a/4db6d72b5a774554b1e9fd48bcfb9ffb 2024-12-06T05:25:03,543 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/a/4db6d72b5a774554b1e9fd48bcfb9ffb, entries=10, sequenceid=33, filesize=5.8 K 2024-12-06T05:25:03,544 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/b/74ddeca3a27d461aa8cf5479ec41615a as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/b/74ddeca3a27d461aa8cf5479ec41615a 2024-12-06T05:25:03,550 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/b/74ddeca3a27d461aa8cf5479ec41615a, entries=10, sequenceid=33, filesize=5.8 K 2024-12-06T05:25:03,551 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/c/78fae4b3efc44e779da75150df447d36 as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/c/78fae4b3efc44e779da75150df447d36 2024-12-06T05:25:03,558 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/c/78fae4b3efc44e779da75150df447d36, entries=10, sequenceid=33, filesize=5.8 K 2024-12-06T05:25:03,559 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for ea3e4b760d1e140747a9cd737237c04c in 97ms, sequenceid=33, compaction requested=false 2024-12-06T05:25:03,560 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ea3e4b760d1e140747a9cd737237c04c: 2024-12-06T05:25:03,560 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea3e4b760d1e140747a9cd737237c04c, disabling compactions & flushes 2024-12-06T05:25:03,560 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. 2024-12-06T05:25:03,560 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. 2024-12-06T05:25:03,560 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. after waiting 0 ms 2024-12-06T05:25:03,560 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. 2024-12-06T05:25:03,562 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. 2024-12-06T05:25:03,562 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea3e4b760d1e140747a9cd737237c04c: Waiting for close lock at 1733462703560Disabling compacts and flushes for region at 1733462703560Disabling writes for close at 1733462703560Writing region close event to WAL at 1733462703561 (+1 ms)Closed at 1733462703561 2024-12-06T05:25:03,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741894_1072 (size=3384) 2024-12-06T05:25:03,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741894_1072 (size=3384) 2024-12-06T05:25:03,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741894_1072 (size=3384) 2024-12-06T05:25:03,568 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/b/74ddeca3a27d461aa8cf5479ec41615a to hdfs://localhost:45541/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/b/74ddeca3a27d461aa8cf5479ec41615a 2024-12-06T05:25:03,583 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407, size=3.3 K (3384bytes) 2024-12-06T05:25:03,583 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407 2024-12-06T05:25:03,583 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407 after 0ms 2024-12-06T05:25:03,585 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:03,586 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407 took 3ms 2024-12-06T05:25:03,588 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407 so closing down 2024-12-06T05:25:03,589 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:25:03,589 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733462703407.temp 2024-12-06T05:25:03,590 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000003-wal.1733462703407.temp 2024-12-06T05:25:03,591 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:25:03,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741898_1076 (size=2944) 2024-12-06T05:25:03,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741898_1076 (size=2944) 2024-12-06T05:25:03,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741898_1076 (size=2944) 2024-12-06T05:25:03,597 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000003-wal.1733462703407.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-06T05:25:03,599 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000003-wal.1733462703407.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000032 2024-12-06T05:25:03,599 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407, size=3.3 K, length=3384, corrupted=false, cancelled=false 2024-12-06T05:25:03,599 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407, journal: Splitting hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407, size=3.3 K (3384bytes) at 1733462703583Finishing writing output for hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407 so closing down at 1733462703588 (+5 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000003-wal.1733462703407.temp at 1733462703590 (+2 ms)3 split writer threads finished at 1733462703591 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000003-wal.1733462703407.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733462703597 (+6 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000003-wal.1733462703407.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000032 at 1733462703599 (+2 ms)Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407, size=3.3 K, length=3384, corrupted=false, cancelled=false at 1733462703599 2024-12-06T05:25:03,600 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703407 to hdfs://localhost:45541/hbase/oldWALs/wal.1733462703407 2024-12-06T05:25:03,601 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000032 2024-12-06T05:25:03,601 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:03,603 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:03,614 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703603, exclude list is [], retry=0 2024-12-06T05:25:03,616 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:03,617 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:03,617 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:03,619 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703603 2024-12-06T05:25:03,619 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:25:03,619 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea3e4b760d1e140747a9cd737237c04c, NAME => 'testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:03,619 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:03,619 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,619 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,621 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,621 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea3e4b760d1e140747a9cd737237c04c columnFamilyName a 2024-12-06T05:25:03,621 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,627 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/a/4db6d72b5a774554b1e9fd48bcfb9ffb 2024-12-06T05:25:03,627 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(327): Store=ea3e4b760d1e140747a9cd737237c04c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,627 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,628 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea3e4b760d1e140747a9cd737237c04c columnFamilyName b 2024-12-06T05:25:03,628 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,629 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(327): Store=ea3e4b760d1e140747a9cd737237c04c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,629 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,629 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea3e4b760d1e140747a9cd737237c04c columnFamilyName c 2024-12-06T05:25:03,630 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,635 DEBUG [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/c/78fae4b3efc44e779da75150df447d36 2024-12-06T05:25:03,635 INFO [StoreOpener-ea3e4b760d1e140747a9cd737237c04c-1 {}] regionserver.HStore(327): Store=ea3e4b760d1e140747a9cd737237c04c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,636 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,636 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,638 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,639 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000032 2024-12-06T05:25:03,642 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:03,643 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000032 2024-12-06T05:25:03,643 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea3e4b760d1e140747a9cd737237c04c 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-06T05:25:03,657 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/b/fa2131bd37a540c9be24d50ad5a58c6b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733462703447/Put/seqid=0 2024-12-06T05:25:03,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741900_1078 (size=5958) 2024-12-06T05:25:03,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741900_1078 (size=5958) 2024-12-06T05:25:03,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741900_1078 (size=5958) 2024-12-06T05:25:03,664 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/b/fa2131bd37a540c9be24d50ad5a58c6b 2024-12-06T05:25:03,671 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/.tmp/b/fa2131bd37a540c9be24d50ad5a58c6b as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/b/fa2131bd37a540c9be24d50ad5a58c6b 2024-12-06T05:25:03,678 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/b/fa2131bd37a540c9be24d50ad5a58c6b, entries=10, sequenceid=32, filesize=5.8 K 2024-12-06T05:25:03,678 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ea3e4b760d1e140747a9cd737237c04c in 35ms, sequenceid=32, compaction requested=false; wal=null 2024-12-06T05:25:03,679 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/0000000000000000032 2024-12-06T05:25:03,680 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,680 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,681 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:03,682 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea3e4b760d1e140747a9cd737237c04c 2024-12-06T05:25:03,685 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/ea3e4b760d1e140747a9cd737237c04c/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-06T05:25:03,686 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea3e4b760d1e140747a9cd737237c04c; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67413758, jitterRate=0.004543274641036987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:03,686 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea3e4b760d1e140747a9cd737237c04c: Writing region info on filesystem at 1733462703619Initializing all the Stores at 1733462703620 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703620Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703620Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703620Obtaining lock to block concurrent updates at 1733462703644 (+24 ms)Preparing flush snapshotting stores in ea3e4b760d1e140747a9cd737237c04c at 1733462703644Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733462703644Flushing stores of testReplayEditsWrittenViaHRegion,,1733462703352.ea3e4b760d1e140747a9cd737237c04c. at 1733462703644Flushing ea3e4b760d1e140747a9cd737237c04c/b: creating writer at 1733462703644Flushing ea3e4b760d1e140747a9cd737237c04c/b: appending metadata at 1733462703656 (+12 ms)Flushing ea3e4b760d1e140747a9cd737237c04c/b: closing flushed file at 1733462703656Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d763079: reopening flushed file at 1733462703670 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ea3e4b760d1e140747a9cd737237c04c in 35ms, sequenceid=32, compaction requested=false; wal=null at 1733462703678 (+8 ms)Cleaning up temporary data from old regions at 1733462703680 (+2 ms)Region opened successfully at 1733462703686 (+6 ms) 2024-12-06T05:25:03,706 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=413 (was 404) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:39382 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:48560 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:38030 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:47490 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:33132 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:45428 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1121 (was 1045) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 272), ProcessCount=11 (was 11), AvailableMemoryMB=7452 (was 7456) 2024-12-06T05:25:03,707 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1121 is superior to 1024 2024-12-06T05:25:03,717 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=413, OpenFileDescriptor=1121, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=7452 2024-12-06T05:25:03,717 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1121 is superior to 1024 2024-12-06T05:25:03,729 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:25:03,730 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:25:03,731 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:25:03,733 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-62182439, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/hregion-62182439, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:03,744 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-62182439/hregion-62182439.1733462703733, exclude list is [], retry=0 2024-12-06T05:25:03,746 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:03,746 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:03,747 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:03,748 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-62182439/hregion-62182439.1733462703733 2024-12-06T05:25:03,748 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:25:03,749 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7b23daeca079ec047934a4ed0c2f80da, NAME => 'testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/hbase 2024-12-06T05:25:03,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741902_1080 (size=68) 2024-12-06T05:25:03,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741902_1080 (size=68) 2024-12-06T05:25:03,757 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:03,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741902_1080 (size=68) 2024-12-06T05:25:03,759 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,760 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b23daeca079ec047934a4ed0c2f80da columnFamilyName a 2024-12-06T05:25:03,760 DEBUG [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,760 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(327): Store=7b23daeca079ec047934a4ed0c2f80da/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,760 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,762 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b23daeca079ec047934a4ed0c2f80da columnFamilyName b 2024-12-06T05:25:03,762 DEBUG [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,762 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(327): Store=7b23daeca079ec047934a4ed0c2f80da/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,762 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,763 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b23daeca079ec047934a4ed0c2f80da columnFamilyName c 2024-12-06T05:25:03,763 DEBUG [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,764 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(327): Store=7b23daeca079ec047934a4ed0c2f80da/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,764 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,765 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,765 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,766 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,766 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,766 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:03,767 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,769 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:25:03,770 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b23daeca079ec047934a4ed0c2f80da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66377861, jitterRate=-0.010892793536186218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:03,770 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b23daeca079ec047934a4ed0c2f80da: Writing region info on filesystem at 1733462703758Initializing all the Stores at 1733462703758Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703758Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703758Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703758Cleaning up temporary data from old regions at 1733462703766 (+8 ms)Region opened successfully at 1733462703770 (+4 ms) 2024-12-06T05:25:03,770 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7b23daeca079ec047934a4ed0c2f80da, disabling compactions & flushes 2024-12-06T05:25:03,770 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. 2024-12-06T05:25:03,770 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. 2024-12-06T05:25:03,770 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. after waiting 0 ms 2024-12-06T05:25:03,770 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. 2024-12-06T05:25:03,771 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. 2024-12-06T05:25:03,771 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7b23daeca079ec047934a4ed0c2f80da: Waiting for close lock at 1733462703770Disabling compacts and flushes for region at 1733462703770Disabling writes for close at 1733462703770Writing region close event to WAL at 1733462703771 (+1 ms)Closed at 1733462703771 2024-12-06T05:25:03,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741901_1079 (size=95) 2024-12-06T05:25:03,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741901_1079 (size=95) 2024-12-06T05:25:03,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741901_1079 (size=95) 2024-12-06T05:25:03,775 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:25:03,775 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-62182439:(num 1733462703733) 2024-12-06T05:25:03,775 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:03,777 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:03,792 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777, exclude list is [], retry=0 2024-12-06T05:25:03,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:03,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:03,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:03,797 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777 2024-12-06T05:25:03,797 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:25:03,848 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b23daeca079ec047934a4ed0c2f80da, NAME => 'testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:03,850 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,850 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:03,850 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,850 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,852 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,853 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b23daeca079ec047934a4ed0c2f80da columnFamilyName a 2024-12-06T05:25:03,853 DEBUG [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,853 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(327): Store=7b23daeca079ec047934a4ed0c2f80da/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,854 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,855 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b23daeca079ec047934a4ed0c2f80da columnFamilyName b 2024-12-06T05:25:03,855 DEBUG [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,855 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(327): Store=7b23daeca079ec047934a4ed0c2f80da/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,855 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,856 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b23daeca079ec047934a4ed0c2f80da columnFamilyName c 2024-12-06T05:25:03,856 DEBUG [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:03,857 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(327): Store=7b23daeca079ec047934a4ed0c2f80da/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:03,857 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,857 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,859 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,860 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,860 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,861 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:03,862 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,863 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b23daeca079ec047934a4ed0c2f80da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68051979, jitterRate=0.014053508639335632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:03,863 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:03,864 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b23daeca079ec047934a4ed0c2f80da: Running coprocessor pre-open hook at 1733462703850Writing region info on filesystem at 1733462703850Initializing all the Stores at 1733462703851 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703851Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703851Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462703851Cleaning up temporary data from old regions at 1733462703860 (+9 ms)Running coprocessor post-open hooks at 1733462703863 (+3 ms)Region opened successfully at 1733462703864 (+1 ms) 2024-12-06T05:25:03,876 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7b23daeca079ec047934a4ed0c2f80da 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-06T05:25:03,878 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 7b23daeca079ec047934a4ed0c2f80da/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:04,879 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 7b23daeca079ec047934a4ed0c2f80da/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:04,939 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-06T05:25:04,939 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-06T05:25:04,940 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-06T05:25:04,941 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-06T05:25:04,941 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-06T05:25:04,941 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-06T05:25:04,942 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-06T05:25:04,942 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-06T05:25:04,943 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-06T05:25:04,943 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-06T05:25:05,869 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T05:25:05,880 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 7b23daeca079ec047934a4ed0c2f80da/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:06,881 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 7b23daeca079ec047934a4ed0c2f80da/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:07,882 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 7b23daeca079ec047934a4ed0c2f80da/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:08,883 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 7b23daeca079ec047934a4ed0c2f80da/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:09,884 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 7b23daeca079ec047934a4ed0c2f80da/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:10,886 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 7b23daeca079ec047934a4ed0c2f80da/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:11,888 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 7b23daeca079ec047934a4ed0c2f80da/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:12,889 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 7b23daeca079ec047934a4ed0c2f80da/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:12,892 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7b23daeca079ec047934a4ed0c2f80da: 2024-12-06T05:25:12,893 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. 2024-12-06T05:25:12,909 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7b23daeca079ec047934a4ed0c2f80da: 2024-12-06T05:25:12,909 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-06T05:25:12,909 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7b23daeca079ec047934a4ed0c2f80da, disabling compactions & flushes 2024-12-06T05:25:12,909 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. 2024-12-06T05:25:12,909 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. 2024-12-06T05:25:12,909 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. after waiting 0 ms 2024-12-06T05:25:12,909 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. 2024-12-06T05:25:12,910 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. 2024-12-06T05:25:12,910 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. 2024-12-06T05:25:12,910 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7b23daeca079ec047934a4ed0c2f80da: Waiting for close lock at 1733462712909Running coprocessor pre-close hooks at 1733462712909Disabling compacts and flushes for region at 1733462712909Disabling writes for close at 1733462712909Writing region close event to WAL at 1733462712910 (+1 ms)Running coprocessor post-close hooks at 1733462712910Closed at 1733462712910 2024-12-06T05:25:12,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741903_1081 (size=2691) 2024-12-06T05:25:12,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741903_1081 (size=2691) 2024-12-06T05:25:12,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741903_1081 (size=2691) 2024-12-06T05:25:12,928 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777, size=2.6 K (2691bytes) 2024-12-06T05:25:12,928 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777 2024-12-06T05:25:12,928 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777 after 0ms 2024-12-06T05:25:12,930 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:12,930 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777 took 3ms 2024-12-06T05:25:12,933 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777 so closing down 2024-12-06T05:25:12,933 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:25:12,934 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733462703777.temp 2024-12-06T05:25:12,935 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000004-wal.1733462703777.temp 2024-12-06T05:25:12,935 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:25:12,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741904_1082 (size=2094) 2024-12-06T05:25:12,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741904_1082 (size=2094) 2024-12-06T05:25:12,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741904_1082 (size=2094) 2024-12-06T05:25:12,943 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000004-wal.1733462703777.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-06T05:25:12,945 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000004-wal.1733462703777.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000026 2024-12-06T05:25:12,945 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-12-06T05:25:12,945 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777, journal: Splitting hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777, size=2.6 K (2691bytes) at 1733462712928Finishing writing output for hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777 so closing down at 1733462712933 (+5 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000004-wal.1733462703777.temp at 1733462712935 (+2 ms)3 split writer threads finished at 1733462712935Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000004-wal.1733462703777.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733462712943 (+8 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000004-wal.1733462703777.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000026 at 1733462712945 (+2 ms)Processed 23 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1733462712945 2024-12-06T05:25:12,947 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462703777 to hdfs://localhost:45541/hbase/oldWALs/wal.1733462703777 2024-12-06T05:25:12,948 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000026 2024-12-06T05:25:12,948 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:12,949 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:12,961 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462712950, exclude list is [], retry=0 2024-12-06T05:25:12,963 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:12,964 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:12,964 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:12,965 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462712950 2024-12-06T05:25:12,966 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:25:12,966 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b23daeca079ec047934a4ed0c2f80da, NAME => 'testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:12,967 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:12,967 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:12,967 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:12,967 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:12,969 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:12,970 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b23daeca079ec047934a4ed0c2f80da columnFamilyName a 2024-12-06T05:25:12,970 DEBUG [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:12,970 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(327): Store=7b23daeca079ec047934a4ed0c2f80da/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:12,971 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:12,971 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b23daeca079ec047934a4ed0c2f80da columnFamilyName b 2024-12-06T05:25:12,971 DEBUG [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:12,972 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(327): Store=7b23daeca079ec047934a4ed0c2f80da/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:12,972 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:12,973 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b23daeca079ec047934a4ed0c2f80da columnFamilyName c 2024-12-06T05:25:12,973 DEBUG [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:12,973 INFO [StoreOpener-7b23daeca079ec047934a4ed0c2f80da-1 {}] regionserver.HStore(327): Store=7b23daeca079ec047934a4ed0c2f80da/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:12,973 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:12,974 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:12,975 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:12,976 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000026 2024-12-06T05:25:12,978 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:12,979 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000026 2024-12-06T05:25:12,979 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7b23daeca079ec047934a4ed0c2f80da 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-06T05:25:12,993 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/.tmp/a/7dced2def1e24745ab8c20e04af1b9bf is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733462712899/Put/seqid=0 2024-12-06T05:25:12,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741906_1084 (size=5523) 2024-12-06T05:25:12,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741906_1084 (size=5523) 2024-12-06T05:25:12,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741906_1084 (size=5523) 2024-12-06T05:25:13,000 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/.tmp/a/7dced2def1e24745ab8c20e04af1b9bf 2024-12-06T05:25:13,019 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/.tmp/b/b99dcb9797a6471db1580fb06ebaa9d6 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733462712894/Put/seqid=0 2024-12-06T05:25:13,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741907_1085 (size=5524) 2024-12-06T05:25:13,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741907_1085 (size=5524) 2024-12-06T05:25:13,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741907_1085 (size=5524) 2024-12-06T05:25:13,026 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/.tmp/b/b99dcb9797a6471db1580fb06ebaa9d6 2024-12-06T05:25:13,049 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/.tmp/c/fcea6ea22b3944ec9140ce9d2b6d1580 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733462712897/Put/seqid=0 2024-12-06T05:25:13,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741908_1086 (size=5457) 2024-12-06T05:25:13,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741908_1086 (size=5457) 2024-12-06T05:25:13,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741908_1086 (size=5457) 2024-12-06T05:25:13,059 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/.tmp/c/fcea6ea22b3944ec9140ce9d2b6d1580 2024-12-06T05:25:13,067 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/.tmp/a/7dced2def1e24745ab8c20e04af1b9bf as hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/a/7dced2def1e24745ab8c20e04af1b9bf 2024-12-06T05:25:13,073 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/a/7dced2def1e24745ab8c20e04af1b9bf, entries=7, sequenceid=26, filesize=5.4 K 2024-12-06T05:25:13,074 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/.tmp/b/b99dcb9797a6471db1580fb06ebaa9d6 as hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/b/b99dcb9797a6471db1580fb06ebaa9d6 2024-12-06T05:25:13,081 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/b/b99dcb9797a6471db1580fb06ebaa9d6, entries=7, sequenceid=26, filesize=5.4 K 2024-12-06T05:25:13,082 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/.tmp/c/fcea6ea22b3944ec9140ce9d2b6d1580 as hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/c/fcea6ea22b3944ec9140ce9d2b6d1580 2024-12-06T05:25:13,089 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/c/fcea6ea22b3944ec9140ce9d2b6d1580, entries=6, sequenceid=26, filesize=5.3 K 2024-12-06T05:25:13,089 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 7b23daeca079ec047934a4ed0c2f80da in 110ms, sequenceid=26, compaction requested=false; wal=null 2024-12-06T05:25:13,090 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/0000000000000000026 2024-12-06T05:25:13,092 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:13,092 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:13,093 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:13,095 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:13,097 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsAfterAbortingFlush/7b23daeca079ec047934a4ed0c2f80da/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-06T05:25:13,099 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7b23daeca079ec047934a4ed0c2f80da; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60658178, jitterRate=-0.09612271189689636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:13,099 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b23daeca079ec047934a4ed0c2f80da 2024-12-06T05:25:13,100 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7b23daeca079ec047934a4ed0c2f80da: Running coprocessor pre-open hook at 1733462712967Writing region info on filesystem at 1733462712967Initializing all the Stores at 1733462712968 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462712968Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462712968Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462712968Obtaining lock to block concurrent updates at 1733462712979 (+11 ms)Preparing flush snapshotting stores in 7b23daeca079ec047934a4ed0c2f80da at 1733462712979Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733462712979Flushing stores of testReplayEditsAfterAbortingFlush,,1733462703729.7b23daeca079ec047934a4ed0c2f80da. at 1733462712979Flushing 7b23daeca079ec047934a4ed0c2f80da/a: creating writer at 1733462712979Flushing 7b23daeca079ec047934a4ed0c2f80da/a: appending metadata at 1733462712992 (+13 ms)Flushing 7b23daeca079ec047934a4ed0c2f80da/a: closing flushed file at 1733462712992Flushing 7b23daeca079ec047934a4ed0c2f80da/b: creating writer at 1733462713005 (+13 ms)Flushing 7b23daeca079ec047934a4ed0c2f80da/b: appending metadata at 1733462713019 (+14 ms)Flushing 7b23daeca079ec047934a4ed0c2f80da/b: closing flushed file at 1733462713019Flushing 7b23daeca079ec047934a4ed0c2f80da/c: creating writer at 1733462713032 (+13 ms)Flushing 7b23daeca079ec047934a4ed0c2f80da/c: appending metadata at 1733462713048 (+16 ms)Flushing 7b23daeca079ec047934a4ed0c2f80da/c: closing flushed file at 1733462713048Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71e995b7: reopening flushed file at 1733462713066 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ef85d90: reopening flushed file at 1733462713073 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51990004: reopening flushed file at 1733462713081 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 7b23daeca079ec047934a4ed0c2f80da in 110ms, sequenceid=26, compaction requested=false; wal=null at 1733462713089 (+8 ms)Cleaning up temporary data from old regions at 1733462713092 (+3 ms)Running coprocessor post-open hooks at 1733462713099 (+7 ms)Region opened successfully at 1733462713100 (+1 ms) 2024-12-06T05:25:13,122 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=410 (was 413), OpenFileDescriptor=1177 (was 1121) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=238 (was 272), ProcessCount=11 (was 11), AvailableMemoryMB=7430 (was 7452) 2024-12-06T05:25:13,122 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1177 is superior to 1024 2024-12-06T05:25:13,136 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=410, OpenFileDescriptor=1177, MaxFileDescriptor=1048576, SystemLoadAverage=238, ProcessCount=11, AvailableMemoryMB=7427 2024-12-06T05:25:13,136 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1177 is superior to 1024 2024-12-06T05:25:13,158 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:25:13,160 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:25:13,161 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:25:13,164 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-59184427, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/hregion-59184427, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:13,175 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-59184427/hregion-59184427.1733462713164, exclude list is [], retry=0 2024-12-06T05:25:13,179 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:13,179 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:13,179 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:13,182 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-59184427/hregion-59184427.1733462713164 2024-12-06T05:25:13,182 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:25:13,182 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ca2a436c578a1245a6a0b0e60889d099, NAME => 'testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/hbase 2024-12-06T05:25:13,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741910_1088 (size=61) 2024-12-06T05:25:13,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741910_1088 (size=61) 2024-12-06T05:25:13,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741910_1088 (size=61) 2024-12-06T05:25:13,191 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:13,192 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,193 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca2a436c578a1245a6a0b0e60889d099 columnFamilyName a 2024-12-06T05:25:13,193 DEBUG [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:13,193 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] regionserver.HStore(327): Store=ca2a436c578a1245a6a0b0e60889d099/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:13,194 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,195 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,195 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,196 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,198 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:25:13,198 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca2a436c578a1245a6a0b0e60889d099; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61687874, jitterRate=-0.0807790458202362}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:25:13,199 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca2a436c578a1245a6a0b0e60889d099: Writing region info on filesystem at 1733462713191Initializing all the Stores at 1733462713191Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462713191Cleaning up temporary data from old regions at 1733462713195 (+4 ms)Region opened successfully at 1733462713199 (+4 ms) 2024-12-06T05:25:13,199 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ca2a436c578a1245a6a0b0e60889d099, disabling compactions & flushes 2024-12-06T05:25:13,199 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. 2024-12-06T05:25:13,199 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. 2024-12-06T05:25:13,199 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. after waiting 0 ms 2024-12-06T05:25:13,199 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. 2024-12-06T05:25:13,200 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. 2024-12-06T05:25:13,200 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ca2a436c578a1245a6a0b0e60889d099: Waiting for close lock at 1733462713199Disabling compacts and flushes for region at 1733462713199Disabling writes for close at 1733462713199Writing region close event to WAL at 1733462713200 (+1 ms)Closed at 1733462713200 2024-12-06T05:25:13,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741909_1087 (size=95) 2024-12-06T05:25:13,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741909_1087 (size=95) 2024-12-06T05:25:13,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741909_1087 (size=95) 2024-12-06T05:25:13,205 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:25:13,205 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-59184427:(num 1733462713164) 2024-12-06T05:25:13,206 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:13,207 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:13,225 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208, exclude list is [], retry=0 2024-12-06T05:25:13,228 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:13,228 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:13,228 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:13,230 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208 2024-12-06T05:25:13,230 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:25:13,230 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ca2a436c578a1245a6a0b0e60889d099, NAME => 'testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:13,230 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:13,230 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,231 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,232 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,233 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca2a436c578a1245a6a0b0e60889d099 columnFamilyName a 2024-12-06T05:25:13,233 DEBUG [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:13,234 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] regionserver.HStore(327): Store=ca2a436c578a1245a6a0b0e60889d099/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:13,234 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,235 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,236 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,236 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,236 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,238 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,239 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca2a436c578a1245a6a0b0e60889d099; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69148553, jitterRate=0.030393734574317932}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:25:13,239 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca2a436c578a1245a6a0b0e60889d099: Writing region info on filesystem at 1733462713231Initializing all the Stores at 1733462713232 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462713232Cleaning up temporary data from old regions at 1733462713236 (+4 ms)Region opened successfully at 1733462713239 (+3 ms) 2024-12-06T05:25:13,247 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ca2a436c578a1245a6a0b0e60889d099, disabling compactions & flushes 2024-12-06T05:25:13,247 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. 2024-12-06T05:25:13,247 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. 2024-12-06T05:25:13,247 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. after waiting 0 ms 2024-12-06T05:25:13,247 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. 2024-12-06T05:25:13,248 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. 2024-12-06T05:25:13,248 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. 2024-12-06T05:25:13,248 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ca2a436c578a1245a6a0b0e60889d099: Waiting for close lock at 1733462713247Disabling compacts and flushes for region at 1733462713247Disabling writes for close at 1733462713247Writing region close event to WAL at 1733462713248 (+1 ms)Closed at 1733462713248 2024-12-06T05:25:13,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741911_1089 (size=1050) 2024-12-06T05:25:13,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741911_1089 (size=1050) 2024-12-06T05:25:13,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741911_1089 (size=1050) 2024-12-06T05:25:13,269 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208, size=1.0 K (1050bytes) 2024-12-06T05:25:13,269 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208 2024-12-06T05:25:13,269 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208 after 0ms 2024-12-06T05:25:13,271 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:13,272 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208 took 4ms 2024-12-06T05:25:13,274 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208 so closing down 2024-12-06T05:25:13,274 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:25:13,275 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733462713208.temp 2024-12-06T05:25:13,276 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000003-wal.1733462713208.temp 2024-12-06T05:25:13,276 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:25:13,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741912_1090 (size=1050) 2024-12-06T05:25:13,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741912_1090 (size=1050) 2024-12-06T05:25:13,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741912_1090 (size=1050) 2024-12-06T05:25:13,287 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000003-wal.1733462713208.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-06T05:25:13,289 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000003-wal.1733462713208.temp to hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000012 2024-12-06T05:25:13,289 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-12-06T05:25:13,289 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208, journal: Splitting hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208, size=1.0 K (1050bytes) at 1733462713269Finishing writing output for hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208 so closing down at 1733462713274 (+5 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000003-wal.1733462713208.temp at 1733462713276 (+2 ms)3 split writer threads finished at 1733462713276Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000003-wal.1733462713208.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733462713287 (+11 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000003-wal.1733462713208.temp to hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000012 at 1733462713289 (+2 ms)Processed 10 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1733462713289 2024-12-06T05:25:13,291 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713208 to hdfs://localhost:45541/hbase/oldWALs/wal.1733462713208 2024-12-06T05:25:13,292 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000012 2024-12-06T05:25:13,296 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:13,593 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:13,595 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:13,606 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713595, exclude list is [], retry=0 2024-12-06T05:25:13,608 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:13,608 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:13,608 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:13,610 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713595 2024-12-06T05:25:13,610 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:25:13,610 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ca2a436c578a1245a6a0b0e60889d099, NAME => 'testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:13,611 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:13,611 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,611 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,613 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,615 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca2a436c578a1245a6a0b0e60889d099 columnFamilyName a 2024-12-06T05:25:13,615 DEBUG [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:13,616 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] regionserver.HStore(327): Store=ca2a436c578a1245a6a0b0e60889d099/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:13,616 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,616 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,618 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,619 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000012 2024-12-06T05:25:13,621 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:13,622 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000012 2024-12-06T05:25:13,622 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ca2a436c578a1245a6a0b0e60889d099 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-06T05:25:13,639 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/.tmp/a/c838da929a4143c782be1609e57a8a20 is 79, key is testDatalossWhenInputError/a:x0/1733462713239/Put/seqid=0 2024-12-06T05:25:13,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741914_1092 (size=5808) 2024-12-06T05:25:13,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741914_1092 (size=5808) 2024-12-06T05:25:13,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741914_1092 (size=5808) 2024-12-06T05:25:13,646 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/.tmp/a/c838da929a4143c782be1609e57a8a20 2024-12-06T05:25:13,654 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/.tmp/a/c838da929a4143c782be1609e57a8a20 as hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/a/c838da929a4143c782be1609e57a8a20 2024-12-06T05:25:13,662 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/a/c838da929a4143c782be1609e57a8a20, entries=10, sequenceid=12, filesize=5.7 K 2024-12-06T05:25:13,662 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ca2a436c578a1245a6a0b0e60889d099 in 40ms, sequenceid=12, compaction requested=false; wal=null 2024-12-06T05:25:13,663 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/0000000000000000012 2024-12-06T05:25:13,664 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,664 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,666 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,668 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-06T05:25:13,669 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca2a436c578a1245a6a0b0e60889d099; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61483492, jitterRate=-0.08382457494735718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:25:13,670 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca2a436c578a1245a6a0b0e60889d099: Writing region info on filesystem at 1733462713611Initializing all the Stores at 1733462713613 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462713613Obtaining lock to block concurrent updates at 1733462713622 (+9 ms)Preparing flush snapshotting stores in ca2a436c578a1245a6a0b0e60889d099 at 1733462713622Finished memstore snapshotting testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733462713622Flushing stores of testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099. at 1733462713622Flushing ca2a436c578a1245a6a0b0e60889d099/a: creating writer at 1733462713623 (+1 ms)Flushing ca2a436c578a1245a6a0b0e60889d099/a: appending metadata at 1733462713638 (+15 ms)Flushing ca2a436c578a1245a6a0b0e60889d099/a: closing flushed file at 1733462713639 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47425ef0: reopening flushed file at 1733462713652 (+13 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ca2a436c578a1245a6a0b0e60889d099 in 40ms, sequenceid=12, compaction requested=false; wal=null at 1733462713662 (+10 ms)Cleaning up temporary data from old regions at 1733462713664 (+2 ms)Region opened successfully at 1733462713670 (+6 ms) 2024-12-06T05:25:13,673 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ca2a436c578a1245a6a0b0e60889d099, NAME => 'testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:13,673 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733462713158.ca2a436c578a1245a6a0b0e60889d099.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:13,674 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,674 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,675 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,676 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca2a436c578a1245a6a0b0e60889d099 columnFamilyName a 2024-12-06T05:25:13,676 DEBUG [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:13,683 DEBUG [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/a/c838da929a4143c782be1609e57a8a20 2024-12-06T05:25:13,683 INFO [StoreOpener-ca2a436c578a1245a6a0b0e60889d099-1 {}] regionserver.HStore(327): Store=ca2a436c578a1245a6a0b0e60889d099/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:13,683 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,684 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,685 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,686 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,686 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,688 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca2a436c578a1245a6a0b0e60889d099 2024-12-06T05:25:13,690 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testDatalossWhenInputError/ca2a436c578a1245a6a0b0e60889d099/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-06T05:25:13,691 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca2a436c578a1245a6a0b0e60889d099; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59132643, jitterRate=-0.11885495483875275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-06T05:25:13,691 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca2a436c578a1245a6a0b0e60889d099: Writing region info on filesystem at 1733462713674Initializing all the Stores at 1733462713675 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462713675Cleaning up temporary data from old regions at 1733462713686 (+11 ms)Region opened successfully at 1733462713691 (+5 ms) 2024-12-06T05:25:13,706 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=421 (was 410) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:33080 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:33004 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:37964 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:56960 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:56886 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:37952 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-626149022-172.17.0.3-1733462679169:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1261 (was 1177) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=238 (was 238), ProcessCount=11 (was 11), AvailableMemoryMB=7422 (was 7427) 2024-12-06T05:25:13,706 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1261 is superior to 1024 2024-12-06T05:25:13,717 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=421, OpenFileDescriptor=1261, MaxFileDescriptor=1048576, SystemLoadAverage=238, ProcessCount=11, AvailableMemoryMB=7421 2024-12-06T05:25:13,717 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1261 is superior to 1024 2024-12-06T05:25:13,730 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:25:13,731 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:25:13,731 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:25:13,734 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98399556, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/hregion-98399556, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:13,744 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98399556/hregion-98399556.1733462713734, exclude list is [], retry=0 2024-12-06T05:25:13,747 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:13,747 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:13,747 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:13,749 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98399556/hregion-98399556.1733462713734 2024-12-06T05:25:13,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:25:13,750 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => db669c79f7f5cdcd4c752be115b05d5a, NAME => 'testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/hbase 2024-12-06T05:25:13,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741916_1094 (size=63) 2024-12-06T05:25:13,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741916_1094 (size=63) 2024-12-06T05:25:13,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741916_1094 (size=63) 2024-12-06T05:25:13,758 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:13,760 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,761 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db669c79f7f5cdcd4c752be115b05d5a columnFamilyName a 2024-12-06T05:25:13,761 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:13,761 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(327): Store=db669c79f7f5cdcd4c752be115b05d5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:13,761 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,763 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db669c79f7f5cdcd4c752be115b05d5a columnFamilyName b 2024-12-06T05:25:13,763 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:13,763 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(327): Store=db669c79f7f5cdcd4c752be115b05d5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:13,763 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,765 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db669c79f7f5cdcd4c752be115b05d5a columnFamilyName c 2024-12-06T05:25:13,765 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:13,765 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(327): Store=db669c79f7f5cdcd4c752be115b05d5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:13,765 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,766 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,767 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,768 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,768 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,769 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:13,770 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,772 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:25:13,772 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db669c79f7f5cdcd4c752be115b05d5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71998775, jitterRate=0.07286535203456879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:13,773 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db669c79f7f5cdcd4c752be115b05d5a: Writing region info on filesystem at 1733462713758Initializing all the Stores at 1733462713759 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462713759Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462713759Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462713759Cleaning up temporary data from old regions at 1733462713768 (+9 ms)Region opened successfully at 1733462713773 (+5 ms) 2024-12-06T05:25:13,773 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing db669c79f7f5cdcd4c752be115b05d5a, disabling compactions & flushes 2024-12-06T05:25:13,773 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. 2024-12-06T05:25:13,774 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. 2024-12-06T05:25:13,774 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. after waiting 0 ms 2024-12-06T05:25:13,774 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. 2024-12-06T05:25:13,774 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. 2024-12-06T05:25:13,774 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for db669c79f7f5cdcd4c752be115b05d5a: Waiting for close lock at 1733462713773Disabling compacts and flushes for region at 1733462713773Disabling writes for close at 1733462713774 (+1 ms)Writing region close event to WAL at 1733462713774Closed at 1733462713774 2024-12-06T05:25:13,775 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-06T05:25:13,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741915_1093 (size=95) 2024-12-06T05:25:13,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741915_1093 (size=95) 2024-12-06T05:25:13,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741915_1093 (size=95) 2024-12-06T05:25:13,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:25:13,781 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98399556:(num 1733462713734) 2024-12-06T05:25:13,782 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:13,784 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:13,798 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784, exclude list is [], retry=0 2024-12-06T05:25:13,800 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:13,801 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:13,801 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:13,802 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 2024-12-06T05:25:13,802 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131)] 2024-12-06T05:25:13,802 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => db669c79f7f5cdcd4c752be115b05d5a, NAME => 'testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:13,803 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:13,803 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,803 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,804 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,804 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db669c79f7f5cdcd4c752be115b05d5a columnFamilyName a 2024-12-06T05:25:13,804 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:13,805 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(327): Store=db669c79f7f5cdcd4c752be115b05d5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:13,805 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,805 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db669c79f7f5cdcd4c752be115b05d5a columnFamilyName b 2024-12-06T05:25:13,805 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:13,806 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(327): Store=db669c79f7f5cdcd4c752be115b05d5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:13,806 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,806 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db669c79f7f5cdcd4c752be115b05d5a columnFamilyName c 2024-12-06T05:25:13,806 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:13,807 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(327): Store=db669c79f7f5cdcd4c752be115b05d5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:13,807 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,807 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,808 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,809 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,809 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,810 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:13,811 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:13,811 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db669c79f7f5cdcd4c752be115b05d5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66528512, jitterRate=-0.008647918701171875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:13,812 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db669c79f7f5cdcd4c752be115b05d5a: Writing region info on filesystem at 1733462713803Initializing all the Stores at 1733462713803Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462713803Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462713804 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462713804Cleaning up temporary data from old regions at 1733462713809 (+5 ms)Region opened successfully at 1733462713812 (+3 ms) 2024-12-06T05:25:13,815 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733462713815/Put/seqid=0 2024-12-06T05:25:13,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741918_1096 (size=4875) 2024-12-06T05:25:13,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741918_1096 (size=4875) 2024-12-06T05:25:13,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741918_1096 (size=4875) 2024-12-06T05:25:13,825 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733462713825/Put/seqid=0 2024-12-06T05:25:13,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741919_1097 (size=4875) 2024-12-06T05:25:13,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741919_1097 (size=4875) 2024-12-06T05:25:13,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741919_1097 (size=4875) 2024-12-06T05:25:13,837 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733462713837/Put/seqid=0 2024-12-06T05:25:13,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741920_1098 (size=4875) 2024-12-06T05:25:13,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741920_1098 (size=4875) 2024-12-06T05:25:13,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741920_1098 (size=4875) 2024-12-06T05:25:13,844 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in db669c79f7f5cdcd4c752be115b05d5a/a 2024-12-06T05:25:13,848 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-06T05:25:13,848 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-06T05:25:13,848 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in db669c79f7f5cdcd4c752be115b05d5a/a 2024-12-06T05:25:13,852 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-06T05:25:13,852 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-06T05:25:13,852 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in db669c79f7f5cdcd4c752be115b05d5a/a 2024-12-06T05:25:13,855 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-06T05:25:13,855 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-06T05:25:13,856 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing db669c79f7f5cdcd4c752be115b05d5a 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-06T05:25:13,870 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/.tmp/a/118a1a6f7629442983ea55e49c867a84 is 55, key is testCompactedBulkLoadedFiles/a:a/1733462713812/Put/seqid=0 2024-12-06T05:25:13,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741921_1099 (size=5107) 2024-12-06T05:25:13,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741921_1099 (size=5107) 2024-12-06T05:25:13,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741921_1099 (size=5107) 2024-12-06T05:25:13,877 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/.tmp/a/118a1a6f7629442983ea55e49c867a84 2024-12-06T05:25:13,884 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/.tmp/a/118a1a6f7629442983ea55e49c867a84 as hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/118a1a6f7629442983ea55e49c867a84 2024-12-06T05:25:13,890 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/118a1a6f7629442983ea55e49c867a84, entries=1, sequenceid=4, filesize=5.0 K 2024-12-06T05:25:13,891 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for db669c79f7f5cdcd4c752be115b05d5a in 36ms, sequenceid=4, compaction requested=false 2024-12-06T05:25:13,891 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for db669c79f7f5cdcd4c752be115b05d5a: 2024-12-06T05:25:13,893 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_ 2024-12-06T05:25:13,894 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb84d419e73948f080cd5a7b487164e6_SeqId_4_ 2024-12-06T05:25:13,895 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/bf8c4823e0b741bd94480884ab222714_SeqId_4_ 2024-12-06T05:25:13,896 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile0 into db669c79f7f5cdcd4c752be115b05d5a/a as hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_ - updating store file list. 2024-12-06T05:25:13,908 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-06T05:25:13,908 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_ into db669c79f7f5cdcd4c752be115b05d5a/a 2024-12-06T05:25:13,908 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile0 into db669c79f7f5cdcd4c752be115b05d5a/a (new location: hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_) 2024-12-06T05:25:13,909 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile1 into db669c79f7f5cdcd4c752be115b05d5a/a as hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb84d419e73948f080cd5a7b487164e6_SeqId_4_ - updating store file list. 2024-12-06T05:25:13,914 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for eb84d419e73948f080cd5a7b487164e6_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-06T05:25:13,914 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb84d419e73948f080cd5a7b487164e6_SeqId_4_ into db669c79f7f5cdcd4c752be115b05d5a/a 2024-12-06T05:25:13,914 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile1 into db669c79f7f5cdcd4c752be115b05d5a/a (new location: hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb84d419e73948f080cd5a7b487164e6_SeqId_4_) 2024-12-06T05:25:13,916 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile2 into db669c79f7f5cdcd4c752be115b05d5a/a as hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/bf8c4823e0b741bd94480884ab222714_SeqId_4_ - updating store file list. 2024-12-06T05:25:13,920 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bf8c4823e0b741bd94480884ab222714_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-06T05:25:13,920 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/bf8c4823e0b741bd94480884ab222714_SeqId_4_ into db669c79f7f5cdcd4c752be115b05d5a/a 2024-12-06T05:25:13,920 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45541/hbase/testCompactedBulkLoadedFiles/hfile2 into db669c79f7f5cdcd4c752be115b05d5a/a (new location: hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/bf8c4823e0b741bd94480884ab222714_SeqId_4_) 2024-12-06T05:25:13,926 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-06T05:25:13,926 DEBUG [Time-limited test {}] regionserver.HStore(1541): db669c79f7f5cdcd4c752be115b05d5a/a is initiating major compaction (all files) 2024-12-06T05:25:13,927 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of db669c79f7f5cdcd4c752be115b05d5a/a in testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. 2024-12-06T05:25:13,927 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/118a1a6f7629442983ea55e49c867a84, hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_, hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb84d419e73948f080cd5a7b487164e6_SeqId_4_, hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/bf8c4823e0b741bd94480884ab222714_SeqId_4_] into tmpdir=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/.tmp, totalSize=19.3 K 2024-12-06T05:25:13,927 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 118a1a6f7629442983ea55e49c867a84, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733462713812 2024-12-06T05:25:13,928 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-06T05:25:13,928 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting eb84d419e73948f080cd5a7b487164e6_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-06T05:25:13,928 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bf8c4823e0b741bd94480884ab222714_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-06T05:25:13,940 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/.tmp/a/eb3a5a21bb3d4b95b749a7b433e87f5d is 55, key is testCompactedBulkLoadedFiles/a:a/1733462713812/Put/seqid=0 2024-12-06T05:25:13,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741922_1100 (size=6154) 2024-12-06T05:25:13,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741922_1100 (size=6154) 2024-12-06T05:25:13,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741922_1100 (size=6154) 2024-12-06T05:25:13,954 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/.tmp/a/eb3a5a21bb3d4b95b749a7b433e87f5d as hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb3a5a21bb3d4b95b749a7b433e87f5d 2024-12-06T05:25:13,961 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in db669c79f7f5cdcd4c752be115b05d5a/a of db669c79f7f5cdcd4c752be115b05d5a into eb3a5a21bb3d4b95b749a7b433e87f5d(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-06T05:25:13,961 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for db669c79f7f5cdcd4c752be115b05d5a: 2024-12-06T05:25:13,962 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-06T05:25:13,962 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-06T05:25:13,990 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784, size=0 (0bytes) 2024-12-06T05:25:13,990 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 might be still open, length is 0 2024-12-06T05:25:13,990 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 2024-12-06T05:25:13,991 WARN [IPC Server handler 0 on default port 45541 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-06T05:25:13,991 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 after 0ms 2024-12-06T05:25:14,939 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-06T05:25:14,939 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-06T05:25:14,941 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-06T05:25:14,941 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-06T05:25:16,721 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:38016 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:35849:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38016 dst: /127.0.0.1:35849 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:35849 remote=/127.0.0.1:38016]. Total timeout mills is 60000, 57241 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:16,721 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:56988 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:43097:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56988 dst: /127.0.0.1:43097 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:16,721 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:33130 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45599:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33130 dst: /127.0.0.1:45599 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:16,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741917_1101 (size=1175) 2024-12-06T05:25:16,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741917_1101 (size=1175) 2024-12-06T05:25:17,993 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 after 4001ms 2024-12-06T05:25:17,999 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:17,999 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 took 4009ms 2024-12-06T05:25:18,002 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784; continuing. 2024-12-06T05:25:18,002 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 so closing down 2024-12-06T05:25:18,002 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:25:18,004 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733462713784.temp 2024-12-06T05:25:18,006 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000003-wal.1733462713784.temp 2024-12-06T05:25:18,006 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:25:18,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741923_1102 (size=549) 2024-12-06T05:25:18,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741923_1102 (size=549) 2024-12-06T05:25:18,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741923_1102 (size=549) 2024-12-06T05:25:18,013 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000003-wal.1733462713784.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-06T05:25:18,015 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000003-wal.1733462713784.temp to hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000008 2024-12-06T05:25:18,015 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784, size=0, length=0, corrupted=false, cancelled=false 2024-12-06T05:25:18,015 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784, journal: Splitting hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784, size=0 (0bytes) at 1733462713990Finishing writing output for hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 so closing down at 1733462718002 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000003-wal.1733462713784.temp at 1733462718006 (+4 ms)3 split writer threads finished at 1733462718006Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000003-wal.1733462713784.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733462718013 (+7 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000003-wal.1733462713784.temp to hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000008 at 1733462718015 (+2 ms)Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784, size=0, length=0, corrupted=false, cancelled=false at 1733462718015 2024-12-06T05:25:18,017 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 to hdfs://localhost:45541/hbase/oldWALs/wal.1733462713784 2024-12-06T05:25:18,018 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000008 2024-12-06T05:25:18,018 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:18,021 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:18,033 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462718021, exclude list is [], retry=0 2024-12-06T05:25:18,035 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:18,035 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:18,036 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:18,037 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462718021 2024-12-06T05:25:18,037 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:25:18,038 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => db669c79f7f5cdcd4c752be115b05d5a, NAME => 'testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:18,038 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:18,038 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,038 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,039 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,040 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db669c79f7f5cdcd4c752be115b05d5a columnFamilyName a 2024-12-06T05:25:18,040 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,046 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/118a1a6f7629442983ea55e49c867a84 2024-12-06T05:25:18,049 DEBUG [StoreFileOpener-db669c79f7f5cdcd4c752be115b05d5a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-06T05:25:18,049 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_ 2024-12-06T05:25:18,052 DEBUG [StoreFileOpener-db669c79f7f5cdcd4c752be115b05d5a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bf8c4823e0b741bd94480884ab222714_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-06T05:25:18,052 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/bf8c4823e0b741bd94480884ab222714_SeqId_4_ 2024-12-06T05:25:18,056 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb3a5a21bb3d4b95b749a7b433e87f5d 2024-12-06T05:25:18,060 DEBUG [StoreFileOpener-db669c79f7f5cdcd4c752be115b05d5a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for eb84d419e73948f080cd5a7b487164e6_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-06T05:25:18,060 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb84d419e73948f080cd5a7b487164e6_SeqId_4_ 2024-12-06T05:25:18,060 WARN [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/118a1a6f7629442983ea55e49c867a84 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7842c7ae 2024-12-06T05:25:18,060 WARN [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7842c7ae 2024-12-06T05:25:18,060 WARN [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/bf8c4823e0b741bd94480884ab222714_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7842c7ae 2024-12-06T05:25:18,060 WARN [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb84d419e73948f080cd5a7b487164e6_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7842c7ae 2024-12-06T05:25:18,060 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/118a1a6f7629442983ea55e49c867a84, hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_, hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/bf8c4823e0b741bd94480884ab222714_SeqId_4_, hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb84d419e73948f080cd5a7b487164e6_SeqId_4_] to archive 2024-12-06T05:25:18,061 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-06T05:25:18,063 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/118a1a6f7629442983ea55e49c867a84 to hdfs://localhost:45541/hbase/archive/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/118a1a6f7629442983ea55e49c867a84 2024-12-06T05:25:18,064 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_ to hdfs://localhost:45541/hbase/archive/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_ 2024-12-06T05:25:18,065 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/bf8c4823e0b741bd94480884ab222714_SeqId_4_ to hdfs://localhost:45541/hbase/archive/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/bf8c4823e0b741bd94480884ab222714_SeqId_4_ 2024-12-06T05:25:18,066 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb84d419e73948f080cd5a7b487164e6_SeqId_4_ to hdfs://localhost:45541/hbase/archive/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/a/eb84d419e73948f080cd5a7b487164e6_SeqId_4_ 2024-12-06T05:25:18,066 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(327): Store=db669c79f7f5cdcd4c752be115b05d5a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,066 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,067 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db669c79f7f5cdcd4c752be115b05d5a columnFamilyName b 2024-12-06T05:25:18,067 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,068 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(327): Store=db669c79f7f5cdcd4c752be115b05d5a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,068 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,068 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db669c79f7f5cdcd4c752be115b05d5a columnFamilyName c 2024-12-06T05:25:18,068 DEBUG [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,069 INFO [StoreOpener-db669c79f7f5cdcd4c752be115b05d5a-1 {}] regionserver.HStore(327): Store=db669c79f7f5cdcd4c752be115b05d5a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,069 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,069 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,071 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,071 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000008 2024-12-06T05:25:18,073 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:18,075 DEBUG [Time-limited test {}] regionserver.HRegion(5836): db669c79f7f5cdcd4c752be115b05d5a : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "db669c79f7f5cdcd4c752be115b05d5a" family_name: "a" compaction_input: "118a1a6f7629442983ea55e49c867a84" compaction_input: "5a89cc1dc5d14a579d7e832a6d135a5e_SeqId_4_" compaction_input: "eb84d419e73948f080cd5a7b487164e6_SeqId_4_" compaction_input: "bf8c4823e0b741bd94480884ab222714_SeqId_4_" compaction_output: "eb3a5a21bb3d4b95b749a7b433e87f5d" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-06T05:25:18,075 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-06T05:25:18,075 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000008 2024-12-06T05:25:18,076 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/0000000000000000008 2024-12-06T05:25:18,077 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,077 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,078 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:18,079 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db669c79f7f5cdcd4c752be115b05d5a 2024-12-06T05:25:18,081 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testCompactedBulkLoadedFiles/db669c79f7f5cdcd4c752be115b05d5a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-06T05:25:18,082 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db669c79f7f5cdcd4c752be115b05d5a; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71424157, jitterRate=0.06430287659168243}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:18,082 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db669c79f7f5cdcd4c752be115b05d5a: Writing region info on filesystem at 1733462718038Initializing all the Stores at 1733462718039 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718039Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718039Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718039Cleaning up temporary data from old regions at 1733462718077 (+38 ms)Region opened successfully at 1733462718082 (+5 ms) 2024-12-06T05:25:18,084 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing db669c79f7f5cdcd4c752be115b05d5a, disabling compactions & flushes 2024-12-06T05:25:18,084 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. 2024-12-06T05:25:18,084 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. 2024-12-06T05:25:18,084 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. after waiting 0 ms 2024-12-06T05:25:18,084 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. 2024-12-06T05:25:18,085 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733462713730.db669c79f7f5cdcd4c752be115b05d5a. 2024-12-06T05:25:18,085 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for db669c79f7f5cdcd4c752be115b05d5a: Waiting for close lock at 1733462718084Disabling compacts and flushes for region at 1733462718084Disabling writes for close at 1733462718084Writing region close event to WAL at 1733462718085 (+1 ms)Closed at 1733462718085 2024-12-06T05:25:18,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741924_1103 (size=95) 2024-12-06T05:25:18,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741924_1103 (size=95) 2024-12-06T05:25:18,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741924_1103 (size=95) 2024-12-06T05:25:18,089 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:25:18,089 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733462718021) 2024-12-06T05:25:18,101 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=435 (was 421) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2134431292_22 at /127.0.0.1:38076 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:45541 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2134431292_22 at /127.0.0.1:57048 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2134431292_22 at /127.0.0.1:33200 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:45541 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1342 (was 1261) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=219 (was 238), ProcessCount=11 (was 11), AvailableMemoryMB=7410 (was 7421) 2024-12-06T05:25:18,101 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-12-06T05:25:18,112 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=435, OpenFileDescriptor=1342, MaxFileDescriptor=1048576, SystemLoadAverage=219, ProcessCount=11, AvailableMemoryMB=7409 2024-12-06T05:25:18,112 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-12-06T05:25:18,124 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:25:18,126 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-06T05:25:18,126 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-06T05:25:18,128 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-88193318, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/hregion-88193318, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:18,139 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-88193318/hregion-88193318.1733462718128, exclude list is [], retry=0 2024-12-06T05:25:18,142 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:18,142 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:18,142 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:18,143 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-88193318/hregion-88193318.1733462718128 2024-12-06T05:25:18,144 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:25:18,144 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d1a572d840308eb8cad544822790f849, NAME => 'testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45541/hbase 2024-12-06T05:25:18,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741926_1105 (size=67) 2024-12-06T05:25:18,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741926_1105 (size=67) 2024-12-06T05:25:18,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741926_1105 (size=67) 2024-12-06T05:25:18,153 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:18,154 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,155 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName a 2024-12-06T05:25:18,155 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,155 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,156 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,156 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName b 2024-12-06T05:25:18,157 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,157 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,157 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,158 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName c 2024-12-06T05:25:18,158 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,158 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,159 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,159 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,159 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,160 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,160 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,161 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:18,162 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,164 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-06T05:25:18,164 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d1a572d840308eb8cad544822790f849; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70890671, jitterRate=0.05635331571102142}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:18,164 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d1a572d840308eb8cad544822790f849: Writing region info on filesystem at 1733462718153Initializing all the Stores at 1733462718153Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718153Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718154 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718154Cleaning up temporary data from old regions at 1733462718160 (+6 ms)Region opened successfully at 1733462718164 (+4 ms) 2024-12-06T05:25:18,165 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d1a572d840308eb8cad544822790f849, disabling compactions & flushes 2024-12-06T05:25:18,165 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:18,165 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:18,165 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. after waiting 0 ms 2024-12-06T05:25:18,165 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:18,165 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:18,165 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d1a572d840308eb8cad544822790f849: Waiting for close lock at 1733462718164Disabling compacts and flushes for region at 1733462718164Disabling writes for close at 1733462718165 (+1 ms)Writing region close event to WAL at 1733462718165Closed at 1733462718165 2024-12-06T05:25:18,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741925_1104 (size=95) 2024-12-06T05:25:18,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741925_1104 (size=95) 2024-12-06T05:25:18,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741925_1104 (size=95) 2024-12-06T05:25:18,169 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:25:18,169 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-88193318:(num 1733462718128) 2024-12-06T05:25:18,170 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:18,171 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:18,186 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172, exclude list is [], retry=0 2024-12-06T05:25:18,189 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:18,189 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:18,189 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:18,191 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172 2024-12-06T05:25:18,191 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:25:18,191 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d1a572d840308eb8cad544822790f849, NAME => 'testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:18,191 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:18,191 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,191 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,193 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,193 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName a 2024-12-06T05:25:18,193 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,194 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,194 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,195 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName b 2024-12-06T05:25:18,195 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,195 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,195 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,196 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName c 2024-12-06T05:25:18,196 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,196 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,197 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,197 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,198 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,199 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,199 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,200 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:18,201 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,202 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d1a572d840308eb8cad544822790f849; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73295285, jitterRate=0.09218485653400421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:18,202 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d1a572d840308eb8cad544822790f849: Writing region info on filesystem at 1733462718191Initializing all the Stores at 1733462718192 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718192Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718192Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718192Cleaning up temporary data from old regions at 1733462718199 (+7 ms)Region opened successfully at 1733462718202 (+3 ms) 2024-12-06T05:25:18,209 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d1a572d840308eb8cad544822790f849 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-06T05:25:18,224 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/a/7fb332e7995c454fb6e637ecf9ac6209 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733462718202/Put/seqid=0 2024-12-06T05:25:18,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741928_1107 (size=5958) 2024-12-06T05:25:18,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741928_1107 (size=5958) 2024-12-06T05:25:18,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741928_1107 (size=5958) 2024-12-06T05:25:18,231 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/a/7fb332e7995c454fb6e637ecf9ac6209 2024-12-06T05:25:18,235 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/a/7fb332e7995c454fb6e637ecf9ac6209 as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/a/7fb332e7995c454fb6e637ecf9ac6209 2024-12-06T05:25:18,240 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/a/7fb332e7995c454fb6e637ecf9ac6209, entries=10, sequenceid=13, filesize=5.8 K 2024-12-06T05:25:18,241 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for d1a572d840308eb8cad544822790f849 in 32ms, sequenceid=13, compaction requested=false 2024-12-06T05:25:18,241 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d1a572d840308eb8cad544822790f849: 2024-12-06T05:25:18,257 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d1a572d840308eb8cad544822790f849, disabling compactions & flushes 2024-12-06T05:25:18,257 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:18,257 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:18,257 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. after waiting 0 ms 2024-12-06T05:25:18,257 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:18,258 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:18,258 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:18,258 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d1a572d840308eb8cad544822790f849: Waiting for close lock at 1733462718257Disabling compacts and flushes for region at 1733462718257Disabling writes for close at 1733462718257Writing region close event to WAL at 1733462718258 (+1 ms)Closed at 1733462718258 2024-12-06T05:25:18,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741927_1106 (size=3345) 2024-12-06T05:25:18,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741927_1106 (size=3345) 2024-12-06T05:25:18,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741927_1106 (size=3345) 2024-12-06T05:25:18,273 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172, size=3.3 K (3345bytes) 2024-12-06T05:25:18,273 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172 2024-12-06T05:25:18,274 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172 after 1ms 2024-12-06T05:25:18,275 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:18,276 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172 took 3ms 2024-12-06T05:25:18,277 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172 so closing down 2024-12-06T05:25:18,277 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:25:18,278 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733462718172.temp 2024-12-06T05:25:18,280 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000003-wal.1733462718172.temp 2024-12-06T05:25:18,280 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:25:18,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741929_1108 (size=2944) 2024-12-06T05:25:18,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741929_1108 (size=2944) 2024-12-06T05:25:18,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741929_1108 (size=2944) 2024-12-06T05:25:18,289 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000003-wal.1733462718172.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-06T05:25:18,290 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000003-wal.1733462718172.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000035 2024-12-06T05:25:18,291 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172, size=3.3 K, length=3345, corrupted=false, cancelled=false 2024-12-06T05:25:18,291 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172, journal: Splitting hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172, size=3.3 K (3345bytes) at 1733462718273Finishing writing output for hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172 so closing down at 1733462718277 (+4 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000003-wal.1733462718172.temp at 1733462718280 (+3 ms)3 split writer threads finished at 1733462718280Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000003-wal.1733462718172.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733462718289 (+9 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000003-wal.1733462718172.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000035 at 1733462718290 (+1 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172, size=3.3 K, length=3345, corrupted=false, cancelled=false at 1733462718291 (+1 ms) 2024-12-06T05:25:18,292 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718172 to hdfs://localhost:45541/hbase/oldWALs/wal.1733462718172 2024-12-06T05:25:18,293 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000035 2024-12-06T05:25:18,293 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:18,294 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:18,308 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295, exclude list is [], retry=0 2024-12-06T05:25:18,310 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:18,311 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:18,311 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:18,312 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 2024-12-06T05:25:18,312 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46623:46623),(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:39677:39677)] 2024-12-06T05:25:18,312 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d1a572d840308eb8cad544822790f849, NAME => 'testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849.', STARTKEY => '', ENDKEY => ''} 2024-12-06T05:25:18,313 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:18,313 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,313 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,314 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,315 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName a 2024-12-06T05:25:18,315 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,319 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/a/7fb332e7995c454fb6e637ecf9ac6209 2024-12-06T05:25:18,319 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,320 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,321 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName b 2024-12-06T05:25:18,321 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,321 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,322 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,323 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName c 2024-12-06T05:25:18,323 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:18,323 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:18,323 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,324 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,326 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,327 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000035 2024-12-06T05:25:18,329 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:18,330 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000035 2024-12-06T05:25:18,331 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d1a572d840308eb8cad544822790f849 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-06T05:25:18,345 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/b/d6ad2fc267bf4fbc864865ac877a9b68 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733462718242/Put/seqid=0 2024-12-06T05:25:18,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741931_1110 (size=5958) 2024-12-06T05:25:18,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741931_1110 (size=5958) 2024-12-06T05:25:18,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741931_1110 (size=5958) 2024-12-06T05:25:18,351 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/b/d6ad2fc267bf4fbc864865ac877a9b68 2024-12-06T05:25:18,370 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/c/3de634c99b884bc2b0c8bf8fc8ac2da7 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733462718249/Put/seqid=0 2024-12-06T05:25:18,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741932_1111 (size=5958) 2024-12-06T05:25:18,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741932_1111 (size=5958) 2024-12-06T05:25:18,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741932_1111 (size=5958) 2024-12-06T05:25:18,376 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/c/3de634c99b884bc2b0c8bf8fc8ac2da7 2024-12-06T05:25:18,382 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/b/d6ad2fc267bf4fbc864865ac877a9b68 as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/b/d6ad2fc267bf4fbc864865ac877a9b68 2024-12-06T05:25:18,388 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/b/d6ad2fc267bf4fbc864865ac877a9b68, entries=10, sequenceid=35, filesize=5.8 K 2024-12-06T05:25:18,389 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/c/3de634c99b884bc2b0c8bf8fc8ac2da7 as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/c/3de634c99b884bc2b0c8bf8fc8ac2da7 2024-12-06T05:25:18,394 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/c/3de634c99b884bc2b0c8bf8fc8ac2da7, entries=10, sequenceid=35, filesize=5.8 K 2024-12-06T05:25:18,394 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d1a572d840308eb8cad544822790f849 in 64ms, sequenceid=35, compaction requested=false; wal=null 2024-12-06T05:25:18,395 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000035 2024-12-06T05:25:18,396 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,396 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,397 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:18,398 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:18,400 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-06T05:25:18,401 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d1a572d840308eb8cad544822790f849; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63043712, jitterRate=-0.06057548522949219}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:18,401 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d1a572d840308eb8cad544822790f849: Writing region info on filesystem at 1733462718313Initializing all the Stores at 1733462718314 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718314Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718314Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462718314Obtaining lock to block concurrent updates at 1733462718331 (+17 ms)Preparing flush snapshotting stores in d1a572d840308eb8cad544822790f849 at 1733462718331Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733462718331Flushing stores of testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. at 1733462718331Flushing d1a572d840308eb8cad544822790f849/b: creating writer at 1733462718331Flushing d1a572d840308eb8cad544822790f849/b: appending metadata at 1733462718344 (+13 ms)Flushing d1a572d840308eb8cad544822790f849/b: closing flushed file at 1733462718344Flushing d1a572d840308eb8cad544822790f849/c: creating writer at 1733462718356 (+12 ms)Flushing d1a572d840308eb8cad544822790f849/c: appending metadata at 1733462718370 (+14 ms)Flushing d1a572d840308eb8cad544822790f849/c: closing flushed file at 1733462718370Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@360ae078: reopening flushed file at 1733462718381 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@751620d2: reopening flushed file at 1733462718388 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d1a572d840308eb8cad544822790f849 in 64ms, sequenceid=35, compaction requested=false; wal=null at 1733462718394 (+6 ms)Cleaning up temporary data from old regions at 1733462718396 (+2 ms)Region opened successfully at 1733462718401 (+5 ms) 2024-12-06T05:25:18,434 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-06T05:25:18,463 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295, size=0 (0bytes) 2024-12-06T05:25:18,463 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 might be still open, length is 0 2024-12-06T05:25:18,463 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 2024-12-06T05:25:18,463 WARN [IPC Server handler 3 on default port 45541 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-06T05:25:18,464 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 after 0ms 2024-12-06T05:25:19,721 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:57120 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:43097:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57120 dst: /127.0.0.1:43097 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43097 remote=/127.0.0.1:57120]. Total timeout mills is 60000, 58705 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:19,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:33272 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45599:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33272 dst: /127.0.0.1:45599 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:19,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2022982962_22 at /127.0.0.1:38136 [Receiving block BP-626149022-172.17.0.3-1733462679169:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:35849:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38136 dst: /127.0.0.1:35849 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:19,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741930_1112 (size=2936) 2024-12-06T05:25:19,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741930_1112 (size=2936) 2024-12-06T05:25:19,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741930_1112 (size=2936) 2024-12-06T05:25:22,465 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 after 4001ms 2024-12-06T05:25:22,470 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:22,470 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 took 4007ms 2024-12-06T05:25:22,475 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295; continuing. 2024-12-06T05:25:22,475 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 so closing down 2024-12-06T05:25:22,475 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-06T05:25:22,477 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733462718295.temp 2024-12-06T05:25:22,478 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000037-wal.1733462718295.temp 2024-12-06T05:25:22,479 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-06T05:25:22,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741933_1113 (size=2944) 2024-12-06T05:25:22,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741933_1113 (size=2944) 2024-12-06T05:25:22,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741933_1113 (size=2944) 2024-12-06T05:25:22,487 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000037-wal.1733462718295.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-06T05:25:22,489 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000037-wal.1733462718295.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000066 2024-12-06T05:25:22,489 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295, size=0, length=0, corrupted=false, cancelled=false 2024-12-06T05:25:22,489 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295, journal: Splitting hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295, size=0 (0bytes) at 1733462718463Finishing writing output for hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 so closing down at 1733462722475 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000037-wal.1733462718295.temp at 1733462722478 (+3 ms)3 split writer threads finished at 1733462722479 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000037-wal.1733462718295.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733462722487 (+8 ms)Rename recovered edits hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000037-wal.1733462718295.temp to hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000066 at 1733462722489 (+2 ms)Processed 30 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295, size=0, length=0, corrupted=false, cancelled=false at 1733462722489 2024-12-06T05:25:22,490 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 to hdfs://localhost:45541/hbase/oldWALs/wal.1733462718295 2024-12-06T05:25:22,491 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000066 2024-12-06T05:25:22,491 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-06T05:25:22,493 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45541/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124, archiveDir=hdfs://localhost:45541/hbase/oldWALs, maxLogs=32 2024-12-06T05:25:22,504 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462722493, exclude list is [], retry=0 2024-12-06T05:25:22,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35849,DS-cec6918e-88b6-4269-bc4b-a51b4702b755,DISK] 2024-12-06T05:25:22,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45599,DS-09a5675f-41c8-4c76-b091-02cb90807939,DISK] 2024-12-06T05:25:22,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43097,DS-20cfe4c8-d3a9-4c4f-a9ea-c480c62b1b8f,DISK] 2024-12-06T05:25:22,510 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462722493 2024-12-06T05:25:22,510 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39677:39677),(127.0.0.1/127.0.0.1:45131:45131),(127.0.0.1/127.0.0.1:46623:46623)] 2024-12-06T05:25:22,511 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-06T05:25:22,512 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:22,513 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName a 2024-12-06T05:25:22,513 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:22,518 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/a/7fb332e7995c454fb6e637ecf9ac6209 2024-12-06T05:25:22,518 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:22,518 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:22,519 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName b 2024-12-06T05:25:22,519 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:22,524 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/b/d6ad2fc267bf4fbc864865ac877a9b68 2024-12-06T05:25:22,524 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:22,524 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d1a572d840308eb8cad544822790f849 2024-12-06T05:25:22,525 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1a572d840308eb8cad544822790f849 columnFamilyName c 2024-12-06T05:25:22,525 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-06T05:25:22,530 DEBUG [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/c/3de634c99b884bc2b0c8bf8fc8ac2da7 2024-12-06T05:25:22,531 INFO [StoreOpener-d1a572d840308eb8cad544822790f849-1 {}] regionserver.HStore(327): Store=d1a572d840308eb8cad544822790f849/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-06T05:25:22,531 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:22,531 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849 2024-12-06T05:25:22,533 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849 2024-12-06T05:25:22,534 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000066 2024-12-06T05:25:22,535 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-06T05:25:22,539 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000066 2024-12-06T05:25:22,539 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d1a572d840308eb8cad544822790f849 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-06T05:25:22,552 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/a/1ed4b55f5ff84599ae485c1581e62af1 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733462718408/Put/seqid=0 2024-12-06T05:25:22,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741935_1115 (size=5958) 2024-12-06T05:25:22,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741935_1115 (size=5958) 2024-12-06T05:25:22,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741935_1115 (size=5958) 2024-12-06T05:25:22,559 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/a/1ed4b55f5ff84599ae485c1581e62af1 2024-12-06T05:25:22,576 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/b/a479237da64d4e5580ec7397eacce6d8 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733462718414/Put/seqid=0 2024-12-06T05:25:22,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741936_1116 (size=5958) 2024-12-06T05:25:22,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741936_1116 (size=5958) 2024-12-06T05:25:22,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741936_1116 (size=5958) 2024-12-06T05:25:23,007 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/b/a479237da64d4e5580ec7397eacce6d8 2024-12-06T05:25:23,033 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/c/e50fd68919ed4cae8b71f527efc50212 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733462718420/Put/seqid=0 2024-12-06T05:25:23,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741937_1117 (size=5958) 2024-12-06T05:25:23,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741937_1117 (size=5958) 2024-12-06T05:25:23,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741937_1117 (size=5958) 2024-12-06T05:25:23,043 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/c/e50fd68919ed4cae8b71f527efc50212 2024-12-06T05:25:23,047 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/a/1ed4b55f5ff84599ae485c1581e62af1 as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/a/1ed4b55f5ff84599ae485c1581e62af1 2024-12-06T05:25:23,052 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/a/1ed4b55f5ff84599ae485c1581e62af1, entries=10, sequenceid=66, filesize=5.8 K 2024-12-06T05:25:23,053 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/b/a479237da64d4e5580ec7397eacce6d8 as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/b/a479237da64d4e5580ec7397eacce6d8 2024-12-06T05:25:23,058 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/b/a479237da64d4e5580ec7397eacce6d8, entries=10, sequenceid=66, filesize=5.8 K 2024-12-06T05:25:23,058 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/.tmp/c/e50fd68919ed4cae8b71f527efc50212 as hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/c/e50fd68919ed4cae8b71f527efc50212 2024-12-06T05:25:23,063 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/c/e50fd68919ed4cae8b71f527efc50212, entries=10, sequenceid=66, filesize=5.8 K 2024-12-06T05:25:23,063 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d1a572d840308eb8cad544822790f849 in 524ms, sequenceid=66, compaction requested=false; wal=null 2024-12-06T05:25:23,064 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/0000000000000000066 2024-12-06T05:25:23,065 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:23,065 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:23,065 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-06T05:25:23,067 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d1a572d840308eb8cad544822790f849 2024-12-06T05:25:23,069 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/hbase/data/default/testReplayEditsWrittenViaHRegion/d1a572d840308eb8cad544822790f849/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-06T05:25:23,070 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d1a572d840308eb8cad544822790f849; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71266621, jitterRate=0.06195540726184845}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-06T05:25:23,070 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d1a572d840308eb8cad544822790f849: Writing region info on filesystem at 1733462722511Initializing all the Stores at 1733462722512 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462722512Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462722512Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733462722512Obtaining lock to block concurrent updates at 1733462722539 (+27 ms)Preparing flush snapshotting stores in d1a572d840308eb8cad544822790f849 at 1733462722539Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733462722539Flushing stores of testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. at 1733462722539Flushing d1a572d840308eb8cad544822790f849/a: creating writer at 1733462722539Flushing d1a572d840308eb8cad544822790f849/a: appending metadata at 1733462722551 (+12 ms)Flushing d1a572d840308eb8cad544822790f849/a: closing flushed file at 1733462722551Flushing d1a572d840308eb8cad544822790f849/b: creating writer at 1733462722563 (+12 ms)Flushing d1a572d840308eb8cad544822790f849/b: appending metadata at 1733462722575 (+12 ms)Flushing d1a572d840308eb8cad544822790f849/b: closing flushed file at 1733462722575Flushing d1a572d840308eb8cad544822790f849/c: creating writer at 1733462723017 (+442 ms)Flushing d1a572d840308eb8cad544822790f849/c: appending metadata at 1733462723032 (+15 ms)Flushing d1a572d840308eb8cad544822790f849/c: closing flushed file at 1733462723032Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6850bcff: reopening flushed file at 1733462723046 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@447fb7c8: reopening flushed file at 1733462723052 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@81f1c9e: reopening flushed file at 1733462723058 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d1a572d840308eb8cad544822790f849 in 524ms, sequenceid=66, compaction requested=false; wal=null at 1733462723063 (+5 ms)Cleaning up temporary data from old regions at 1733462723065 (+2 ms)Region opened successfully at 1733462723070 (+5 ms) 2024-12-06T05:25:23,081 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d1a572d840308eb8cad544822790f849, disabling compactions & flushes 2024-12-06T05:25:23,082 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:23,082 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:23,082 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. after waiting 0 ms 2024-12-06T05:25:23,082 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:23,083 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733462718125.d1a572d840308eb8cad544822790f849. 2024-12-06T05:25:23,083 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d1a572d840308eb8cad544822790f849: Waiting for close lock at 1733462723081Disabling compacts and flushes for region at 1733462723081Disabling writes for close at 1733462723082 (+1 ms)Writing region close event to WAL at 1733462723083 (+1 ms)Closed at 1733462723083 2024-12-06T05:25:23,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741934_1114 (size=95) 2024-12-06T05:25:23,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741934_1114 (size=95) 2024-12-06T05:25:23,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741934_1114 (size=95) 2024-12-06T05:25:23,087 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-06T05:25:23,087 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733462722493) 2024-12-06T05:25:23,102 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=441 (was 435) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:45541 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_755817278_22 at /127.0.0.1:32980 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:45541 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_755817278_22 at /127.0.0.1:46650 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_755817278_22 at /127.0.0.1:49796 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1342) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=201 (was 219), ProcessCount=11 (was 11), AvailableMemoryMB=7366 (was 7409) 2024-12-06T05:25:23,102 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-12-06T05:25:23,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-06T05:25:23,102 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T05:25:23,103 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:25:23,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:25:23,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:25:23,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-06T05:25:23,103 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-06T05:25:23,104 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1088696683, stopped=false 2024-12-06T05:25:23,104 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ec46afd49254,43357,1733462684569 2024-12-06T05:25:23,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T05:25:23,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T05:25:23,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-06T05:25:23,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:25:23,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:25:23,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:25:23,196 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T05:25:23,197 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-06T05:25:23,197 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:25:23,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:25:23,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:25:23,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:25:23,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-06T05:25:23,197 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,43053,1733462685378' ***** 2024-12-06T05:25:23,197 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T05:25:23,198 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ec46afd49254,42449,1733462685474' ***** 2024-12-06T05:25:23,198 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-06T05:25:23,198 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:25:23,198 INFO [RS:1;ec46afd49254:42449 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-06T05:25:23,198 INFO [RS:1;ec46afd49254:42449 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T05:25:23,198 INFO [RS:0;ec46afd49254:43053 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-06T05:25:23,198 INFO [RS:0;ec46afd49254:43053 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T05:25:23,198 INFO [RS:1;ec46afd49254:42449 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-06T05:25:23,199 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(3091): Received CLOSE for 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:23,199 INFO [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(959): stopping server ec46afd49254,42449,1733462685474 2024-12-06T05:25:23,199 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(959): stopping server ec46afd49254,43053,1733462685378 2024-12-06T05:25:23,199 INFO [RS:1;ec46afd49254:42449 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:25:23,199 INFO [RS:0;ec46afd49254:43053 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:25:23,199 INFO [RS:1;ec46afd49254:42449 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ec46afd49254:42449. 2024-12-06T05:25:23,199 INFO [RS:0;ec46afd49254:43053 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ec46afd49254:43053. 2024-12-06T05:25:23,199 DEBUG [RS:0;ec46afd49254:43053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:25:23,199 DEBUG [RS:1;ec46afd49254:42449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-06T05:25:23,199 DEBUG [RS:0;ec46afd49254:43053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:25:23,199 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 55114b43c8aa4ca85a42aa15503261cc, disabling compactions & flushes 2024-12-06T05:25:23,199 DEBUG [RS:1;ec46afd49254:42449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:25:23,199 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:25:23,199 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:23,199 INFO [RS:0;ec46afd49254:43053 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:25:23,199 INFO [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,42449,1733462685474; all regions closed. 2024-12-06T05:25:23,199 INFO [RS:0;ec46afd49254:43053 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:25:23,199 INFO [RS:0;ec46afd49254:43053 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:25:23,199 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-06T05:25:23,199 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:23,199 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. after waiting 0 ms 2024-12-06T05:25:23,199 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-06T05:25:23,200 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:23,200 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-06T05:25:23,200 DEBUG [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 55114b43c8aa4ca85a42aa15503261cc=testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc.} 2024-12-06T05:25:23,200 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-06T05:25:23,200 DEBUG [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 55114b43c8aa4ca85a42aa15503261cc 2024-12-06T05:25:23,200 INFO [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-06T05:25:23,200 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-06T05:25:23,200 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-06T05:25:23,200 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-06T05:25:23,200 INFO [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-06T05:25:23,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741835_1011 (size=95) 2024-12-06T05:25:23,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741835_1011 (size=95) 2024-12-06T05:25:23,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741835_1011 (size=95) 2024-12-06T05:25:23,208 DEBUG [RS:1;ec46afd49254:42449 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/oldWALs 2024-12-06T05:25:23,209 INFO [RS:1;ec46afd49254:42449 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ec46afd49254%2C42449%2C1733462685474:(num 1733462687186) 2024-12-06T05:25:23,209 DEBUG [RS:1;ec46afd49254:42449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:25:23,209 INFO [RS:1;ec46afd49254:42449 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:25:23,209 INFO [RS:1;ec46afd49254:42449 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:25:23,209 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/default/testReplayEditsAfterRegionMovedWithMultiCF/55114b43c8aa4ca85a42aa15503261cc/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-06T05:25:23,209 INFO [RS:1;ec46afd49254:42449 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-06T05:25:23,209 INFO [RS:1;ec46afd49254:42449 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-06T05:25:23,209 INFO [RS:1;ec46afd49254:42449 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-06T05:25:23,209 INFO [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:25:23,209 INFO [RS:1;ec46afd49254:42449 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-06T05:25:23,209 INFO [RS:1;ec46afd49254:42449 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:25:23,210 INFO [RS:1;ec46afd49254:42449 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42449 2024-12-06T05:25:23,210 INFO [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:23,210 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 55114b43c8aa4ca85a42aa15503261cc: Waiting for close lock at 1733462723199Running coprocessor pre-close hooks at 1733462723199Disabling compacts and flushes for region at 1733462723199Disabling writes for close at 1733462723199Writing region close event to WAL at 1733462723204 (+5 ms)Running coprocessor post-close hooks at 1733462723210 (+6 ms)Closed at 1733462723210 2024-12-06T05:25:23,210 DEBUG [RS_CLOSE_REGION-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc. 2024-12-06T05:25:23,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,42449,1733462685474 2024-12-06T05:25:23,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:25:23,217 INFO [RS:1;ec46afd49254:42449 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:25:23,217 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,42449,1733462685474] 2024-12-06T05:25:23,223 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/.tmp/info/f3daa7c9b7bc41569fd7495f40e88ef0 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733462699946.55114b43c8aa4ca85a42aa15503261cc./info:regioninfo/1733462703062/Put/seqid=0 2024-12-06T05:25:23,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741938_1118 (size=8243) 2024-12-06T05:25:23,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741938_1118 (size=8243) 2024-12-06T05:25:23,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741938_1118 (size=8243) 2024-12-06T05:25:23,230 INFO [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/.tmp/info/f3daa7c9b7bc41569fd7495f40e88ef0 2024-12-06T05:25:23,238 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,42449,1733462685474 already deleted, retry=false 2024-12-06T05:25:23,238 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec46afd49254,42449,1733462685474 expired; onlineServers=1 2024-12-06T05:25:23,245 INFO [regionserver/ec46afd49254:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:25:23,249 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/.tmp/ns/29218cef85b74537994c906252a6d7c5 is 43, key is default/ns:d/1733462687709/Put/seqid=0 2024-12-06T05:25:23,249 INFO [regionserver/ec46afd49254:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:25:23,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741939_1119 (size=5153) 2024-12-06T05:25:23,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741939_1119 (size=5153) 2024-12-06T05:25:23,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741939_1119 (size=5153) 2024-12-06T05:25:23,256 INFO [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/.tmp/ns/29218cef85b74537994c906252a6d7c5 2024-12-06T05:25:23,273 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/.tmp/table/1dfdd4c2366c43a6b95ac8e4c8b7a2dd is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733462700370/Put/seqid=0 2024-12-06T05:25:23,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741940_1120 (size=5431) 2024-12-06T05:25:23,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741940_1120 (size=5431) 2024-12-06T05:25:23,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741940_1120 (size=5431) 2024-12-06T05:25:23,280 INFO [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/.tmp/table/1dfdd4c2366c43a6b95ac8e4c8b7a2dd 2024-12-06T05:25:23,287 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/.tmp/info/f3daa7c9b7bc41569fd7495f40e88ef0 as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/info/f3daa7c9b7bc41569fd7495f40e88ef0 2024-12-06T05:25:23,292 INFO [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/info/f3daa7c9b7bc41569fd7495f40e88ef0, entries=18, sequenceid=21, filesize=8.0 K 2024-12-06T05:25:23,293 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/.tmp/ns/29218cef85b74537994c906252a6d7c5 as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/ns/29218cef85b74537994c906252a6d7c5 2024-12-06T05:25:23,298 INFO [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/ns/29218cef85b74537994c906252a6d7c5, entries=2, sequenceid=21, filesize=5.0 K 2024-12-06T05:25:23,300 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/.tmp/table/1dfdd4c2366c43a6b95ac8e4c8b7a2dd as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/table/1dfdd4c2366c43a6b95ac8e4c8b7a2dd 2024-12-06T05:25:23,306 INFO [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/table/1dfdd4c2366c43a6b95ac8e4c8b7a2dd, entries=2, sequenceid=21, filesize=5.3 K 2024-12-06T05:25:23,307 INFO [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 107ms, sequenceid=21, compaction requested=false 2024-12-06T05:25:23,311 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-06T05:25:23,312 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-06T05:25:23,312 INFO [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-06T05:25:23,312 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733462723200Running coprocessor pre-close hooks at 1733462723200Disabling compacts and flushes for region at 1733462723200Disabling writes for close at 1733462723200Obtaining lock to block concurrent updates at 1733462723200Preparing flush snapshotting stores in 1588230740 at 1733462723200Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733462723201 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733462723202 (+1 ms)Flushing 1588230740/info: creating writer at 1733462723202Flushing 1588230740/info: appending metadata at 1733462723222 (+20 ms)Flushing 1588230740/info: closing flushed file at 1733462723222Flushing 1588230740/ns: creating writer at 1733462723236 (+14 ms)Flushing 1588230740/ns: appending metadata at 1733462723248 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733462723248Flushing 1588230740/table: creating writer at 1733462723261 (+13 ms)Flushing 1588230740/table: appending metadata at 1733462723273 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733462723273Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28430c83: reopening flushed file at 1733462723286 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ef59c41: reopening flushed file at 1733462723292 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@531ed598: reopening flushed file at 1733462723299 (+7 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 107ms, sequenceid=21, compaction requested=false at 1733462723307 (+8 ms)Writing region close event to WAL at 1733462723308 (+1 ms)Running coprocessor post-close hooks at 1733462723312 (+4 ms)Closed at 1733462723312 2024-12-06T05:25:23,312 DEBUG [RS_CLOSE_META-regionserver/ec46afd49254:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-06T05:25:23,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:25:23,328 INFO [RS:1;ec46afd49254:42449 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:25:23,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42449-0x101a9196c580002, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:25:23,328 INFO [RS:1;ec46afd49254:42449 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,42449,1733462685474; zookeeper connection closed. 2024-12-06T05:25:23,328 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@29b5102d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@29b5102d 2024-12-06T05:25:23,400 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(976): stopping server ec46afd49254,43053,1733462685378; all regions closed. 2024-12-06T05:25:23,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741836_1012 (size=4715) 2024-12-06T05:25:23,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741836_1012 (size=4715) 2024-12-06T05:25:23,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741836_1012 (size=4715) 2024-12-06T05:25:23,411 DEBUG [RS:0;ec46afd49254:43053 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/oldWALs 2024-12-06T05:25:23,411 INFO [RS:0;ec46afd49254:43053 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ec46afd49254%2C43053%2C1733462685378.meta:.meta(num 1733462687532) 2024-12-06T05:25:23,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741834_1010 (size=2176) 2024-12-06T05:25:23,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741834_1010 (size=2176) 2024-12-06T05:25:23,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741834_1010 (size=2176) 2024-12-06T05:25:23,415 DEBUG [RS:0;ec46afd49254:43053 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/oldWALs 2024-12-06T05:25:23,415 INFO [RS:0;ec46afd49254:43053 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ec46afd49254%2C43053%2C1733462685378:(num 1733462687186) 2024-12-06T05:25:23,415 DEBUG [RS:0;ec46afd49254:43053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-06T05:25:23,416 INFO [RS:0;ec46afd49254:43053 {}] regionserver.LeaseManager(133): Closed leases 2024-12-06T05:25:23,416 INFO [RS:0;ec46afd49254:43053 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:25:23,416 INFO [RS:0;ec46afd49254:43053 {}] hbase.ChoreService(370): Chore service for: regionserver/ec46afd49254:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-06T05:25:23,416 INFO [RS:0;ec46afd49254:43053 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:25:23,416 INFO [regionserver/ec46afd49254:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:25:23,416 INFO [RS:0;ec46afd49254:43053 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43053 2024-12-06T05:25:23,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ec46afd49254,43053,1733462685378 2024-12-06T05:25:23,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-06T05:25:23,427 INFO [RS:0;ec46afd49254:43053 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:25:23,438 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ec46afd49254,43053,1733462685378] 2024-12-06T05:25:23,448 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ec46afd49254,43053,1733462685378 already deleted, retry=false 2024-12-06T05:25:23,449 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ec46afd49254,43053,1733462685378 expired; onlineServers=0 2024-12-06T05:25:23,449 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ec46afd49254,43357,1733462684569' ***** 2024-12-06T05:25:23,449 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-06T05:25:23,449 INFO [M:0;ec46afd49254:43357 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-06T05:25:23,449 INFO [M:0;ec46afd49254:43357 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-06T05:25:23,449 DEBUG [M:0;ec46afd49254:43357 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-06T05:25:23,450 DEBUG [M:0;ec46afd49254:43357 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-06T05:25:23,450 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-06T05:25:23,450 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462686735 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.small.0-1733462686735,5,FailOnTimeoutGroup] 2024-12-06T05:25:23,450 DEBUG [master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462686734 {}] cleaner.HFileCleaner(306): Exit Thread[master/ec46afd49254:0:becomeActiveMaster-HFileCleaner.large.0-1733462686734,5,FailOnTimeoutGroup] 2024-12-06T05:25:23,450 INFO [M:0;ec46afd49254:43357 {}] hbase.ChoreService(370): Chore service for: master/ec46afd49254:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-06T05:25:23,450 INFO [M:0;ec46afd49254:43357 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-06T05:25:23,451 DEBUG [M:0;ec46afd49254:43357 {}] master.HMaster(1795): Stopping service threads 2024-12-06T05:25:23,451 INFO [M:0;ec46afd49254:43357 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-06T05:25:23,451 INFO [M:0;ec46afd49254:43357 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-06T05:25:23,452 INFO [M:0;ec46afd49254:43357 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-06T05:25:23,452 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-06T05:25:23,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-06T05:25:23,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-06T05:25:23,459 DEBUG [M:0;ec46afd49254:43357 {}] zookeeper.ZKUtil(347): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-06T05:25:23,459 WARN [M:0;ec46afd49254:43357 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-06T05:25:23,460 INFO [M:0;ec46afd49254:43357 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/.lastflushedseqids 2024-12-06T05:25:23,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741941_1121 (size=138) 2024-12-06T05:25:23,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741941_1121 (size=138) 2024-12-06T05:25:23,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741941_1121 (size=138) 2024-12-06T05:25:23,472 INFO [M:0;ec46afd49254:43357 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-06T05:25:23,472 INFO [M:0;ec46afd49254:43357 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-06T05:25:23,472 DEBUG [M:0;ec46afd49254:43357 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-06T05:25:23,472 INFO [M:0;ec46afd49254:43357 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:25:23,472 DEBUG [M:0;ec46afd49254:43357 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:25:23,472 DEBUG [M:0;ec46afd49254:43357 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-06T05:25:23,472 DEBUG [M:0;ec46afd49254:43357 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:25:23,472 INFO [M:0;ec46afd49254:43357 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-12-06T05:25:23,487 DEBUG [M:0;ec46afd49254:43357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/205a6e6c5a374c4e9d8f3a4bcc01fab7 is 82, key is hbase:meta,,1/info:regioninfo/1733462687628/Put/seqid=0 2024-12-06T05:25:23,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741942_1122 (size=5672) 2024-12-06T05:25:23,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741942_1122 (size=5672) 2024-12-06T05:25:23,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741942_1122 (size=5672) 2024-12-06T05:25:23,493 INFO [M:0;ec46afd49254:43357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/205a6e6c5a374c4e9d8f3a4bcc01fab7 2024-12-06T05:25:23,510 DEBUG [M:0;ec46afd49254:43357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2ff39d3d17fe4eafa815e4967635c1f9 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733462700375/Put/seqid=0 2024-12-06T05:25:23,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741943_1123 (size=7755) 2024-12-06T05:25:23,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741943_1123 (size=7755) 2024-12-06T05:25:23,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741943_1123 (size=7755) 2024-12-06T05:25:23,516 INFO [M:0;ec46afd49254:43357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2ff39d3d17fe4eafa815e4967635c1f9 2024-12-06T05:25:23,521 INFO [M:0;ec46afd49254:43357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2ff39d3d17fe4eafa815e4967635c1f9 2024-12-06T05:25:23,534 DEBUG [M:0;ec46afd49254:43357 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b3f690073b9844ff8aaff317d52db4de is 69, key is ec46afd49254,42449,1733462685474/rs:state/1733462686851/Put/seqid=0 2024-12-06T05:25:23,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:25:23,538 INFO [RS:0;ec46afd49254:43053 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:25:23,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43053-0x101a9196c580001, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:25:23,538 INFO [RS:0;ec46afd49254:43053 {}] regionserver.HRegionServer(1031): Exiting; stopping=ec46afd49254,43053,1733462685378; zookeeper connection closed. 2024-12-06T05:25:23,539 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ba81c9e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ba81c9e 2024-12-06T05:25:23,539 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-06T05:25:23,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741944_1124 (size=5440) 2024-12-06T05:25:23,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741944_1124 (size=5440) 2024-12-06T05:25:23,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741944_1124 (size=5440) 2024-12-06T05:25:23,541 INFO [M:0;ec46afd49254:43357 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b3f690073b9844ff8aaff317d52db4de 2024-12-06T05:25:23,545 INFO [M:0;ec46afd49254:43357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b3f690073b9844ff8aaff317d52db4de 2024-12-06T05:25:23,546 DEBUG [M:0;ec46afd49254:43357 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/205a6e6c5a374c4e9d8f3a4bcc01fab7 as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/205a6e6c5a374c4e9d8f3a4bcc01fab7 2024-12-06T05:25:23,551 INFO [M:0;ec46afd49254:43357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/205a6e6c5a374c4e9d8f3a4bcc01fab7, entries=8, sequenceid=168, filesize=5.5 K 2024-12-06T05:25:23,552 DEBUG [M:0;ec46afd49254:43357 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2ff39d3d17fe4eafa815e4967635c1f9 as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2ff39d3d17fe4eafa815e4967635c1f9 2024-12-06T05:25:23,557 INFO [M:0;ec46afd49254:43357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2ff39d3d17fe4eafa815e4967635c1f9 2024-12-06T05:25:23,557 INFO [M:0;ec46afd49254:43357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2ff39d3d17fe4eafa815e4967635c1f9, entries=17, sequenceid=168, filesize=7.6 K 2024-12-06T05:25:23,558 DEBUG [M:0;ec46afd49254:43357 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b3f690073b9844ff8aaff317d52db4de as hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b3f690073b9844ff8aaff317d52db4de 2024-12-06T05:25:23,563 INFO [M:0;ec46afd49254:43357 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b3f690073b9844ff8aaff317d52db4de 2024-12-06T05:25:23,563 INFO [M:0;ec46afd49254:43357 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45541/user/jenkins/test-data/2dc41d85-fb23-b5f6-d3a6-27076ef9f727/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b3f690073b9844ff8aaff317d52db4de, entries=3, sequenceid=168, filesize=5.3 K 2024-12-06T05:25:23,564 INFO [M:0;ec46afd49254:43357 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=168, compaction requested=false 2024-12-06T05:25:23,565 INFO [M:0;ec46afd49254:43357 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-06T05:25:23,565 DEBUG [M:0;ec46afd49254:43357 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733462723472Disabling compacts and flushes for region at 1733462723472Disabling writes for close at 1733462723472Obtaining lock to block concurrent updates at 1733462723472Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733462723472Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1733462723473 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733462723473Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733462723474 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733462723486 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733462723486Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733462723497 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733462723509 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733462723509Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733462723521 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733462723534 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733462723534Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fe23cd2: reopening flushed file at 1733462723545 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@413546ad: reopening flushed file at 1733462723552 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fc16b0a: reopening flushed file at 1733462723557 (+5 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=168, compaction requested=false at 1733462723564 (+7 ms)Writing region close event to WAL at 1733462723565 (+1 ms)Closed at 1733462723565 2024-12-06T05:25:23,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43097 is added to blk_1073741830_1006 (size=56618) 2024-12-06T05:25:23,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35849 is added to blk_1073741830_1006 (size=56618) 2024-12-06T05:25:23,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45599 is added to blk_1073741830_1006 (size=56618) 2024-12-06T05:25:23,568 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-06T05:25:23,568 INFO [M:0;ec46afd49254:43357 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-06T05:25:23,568 INFO [M:0;ec46afd49254:43357 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43357 2024-12-06T05:25:23,568 INFO [M:0;ec46afd49254:43357 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-06T05:25:23,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:25:23,675 INFO [M:0;ec46afd49254:43357 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-06T05:25:23,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43357-0x101a9196c580000, quorum=127.0.0.1:63857, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-06T05:25:23,685 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703603 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703603 (inode 16655) Holder DFSClient_NONMAPREDUCE_2022982962_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733462703350/wal.1733462703603 (inode 16655) Holder DFSClient_NONMAPREDUCE_2022982962_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-06T05:25:23,687 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733462695500/wal.1733462695639 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:23,687 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733462718124/wal.1733462718295 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:23,689 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462712950 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462712950 (inode 16678) Holder DFSClient_NONMAPREDUCE_2022982962_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733462703728/wal.1733462712950 (inode 16678) Holder DFSClient_NONMAPREDUCE_2022982962_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-06T05:25:23,690 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733462688769/wal.1733462688846 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:23,693 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713595 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713595 (inode 16704) Holder DFSClient_NONMAPREDUCE_2022982962_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733462713157/wal.1733462713595 (inode 16704) Holder DFSClient_NONMAPREDUCE_2022982962_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-06T05:25:23,693 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733462713729/wal.1733462713784 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-06T05:25:23,696 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal.1733462688615 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal.1733462688615 (inode 16485) Holder DFSClient_NONMAPREDUCE_2022982962_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733462688427/wal.1733462688615 (inode 16485) Holder DFSClient_NONMAPREDUCE_2022982962_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-06T05:25:23,698 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal.1733462688195 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal.1733462688195 (inode 16462) Holder DFSClient_NONMAPREDUCE_2022982962_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733462687965/wal.1733462688195 (inode 16462) Holder DFSClient_NONMAPREDUCE_2022982962_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-06T05:25:23,702 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:25:23,705 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T05:25:23,705 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T05:25:23,705 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T05:25:23,705 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/hadoop.log.dir/,STOPPED} 2024-12-06T05:25:23,708 WARN [BP-626149022-172.17.0.3-1733462679169 heartbeating to localhost/127.0.0.1:45541 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T05:25:23,708 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T05:25:23,708 WARN [BP-626149022-172.17.0.3-1733462679169 heartbeating to localhost/127.0.0.1:45541 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-626149022-172.17.0.3-1733462679169 (Datanode Uuid 9fdc89ee-443c-4adb-80dd-9e8967c80004) service to localhost/127.0.0.1:45541 2024-12-06T05:25:23,708 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T05:25:23,709 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data5/current/BP-626149022-172.17.0.3-1733462679169 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T05:25:23,710 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data6/current/BP-626149022-172.17.0.3-1733462679169 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T05:25:23,710 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T05:25:23,712 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:25:23,713 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T05:25:23,713 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T05:25:23,713 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T05:25:23,713 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/hadoop.log.dir/,STOPPED} 2024-12-06T05:25:23,714 WARN [BP-626149022-172.17.0.3-1733462679169 heartbeating to localhost/127.0.0.1:45541 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T05:25:23,714 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T05:25:23,714 WARN [BP-626149022-172.17.0.3-1733462679169 heartbeating to localhost/127.0.0.1:45541 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-626149022-172.17.0.3-1733462679169 (Datanode Uuid 03cb825b-e297-4874-9359-89ab346321d0) service to localhost/127.0.0.1:45541 2024-12-06T05:25:23,714 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T05:25:23,715 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data3/current/BP-626149022-172.17.0.3-1733462679169 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T05:25:23,715 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data4/current/BP-626149022-172.17.0.3-1733462679169 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T05:25:23,715 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T05:25:23,721 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-06T05:25:23,721 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T05:25:23,721 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T05:25:23,721 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T05:25:23,721 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/hadoop.log.dir/,STOPPED} 2024-12-06T05:25:23,723 WARN [BP-626149022-172.17.0.3-1733462679169 heartbeating to localhost/127.0.0.1:45541 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-06T05:25:23,723 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-06T05:25:23,723 WARN [BP-626149022-172.17.0.3-1733462679169 heartbeating to localhost/127.0.0.1:45541 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-626149022-172.17.0.3-1733462679169 (Datanode Uuid 179633ab-ee81-4fde-98a0-466db3b1812f) service to localhost/127.0.0.1:45541 2024-12-06T05:25:23,723 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-06T05:25:23,723 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data1/current/BP-626149022-172.17.0.3-1733462679169 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T05:25:23,723 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/cluster_6dab2aca-7d8e-221a-8f7f-c8c87c1825e2/data/data2/current/BP-626149022-172.17.0.3-1733462679169 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-06T05:25:23,724 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-06T05:25:23,728 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-06T05:25:23,729 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-06T05:25:23,729 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-06T05:25:23,729 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-06T05:25:23,729 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a597649a-c7d5-e84c-ec66-4a3e78b7f54f/hadoop.log.dir/,STOPPED} 2024-12-06T05:25:23,737 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-06T05:25:23,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down