2024-11-14 04:34:57,674 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-14 04:34:57,684 main DEBUG Took 0.008453 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 04:34:57,685 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 04:34:57,685 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 04:34:57,686 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 04:34:57,687 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,693 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 04:34:57,704 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,705 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,705 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,706 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,707 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,708 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,708 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,709 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,709 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,710 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,710 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,711 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,712 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,712 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,713 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 04:34:57,714 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,714 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 04:34:57,715 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 04:34:57,717 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 04:34:57,718 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 04:34:57,719 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 04:34:57,720 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 04:34:57,720 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 04:34:57,728 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 04:34:57,730 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 04:34:57,732 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 04:34:57,732 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 04:34:57,732 main DEBUG createAppenders(={Console}) 2024-11-14 04:34:57,733 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-14 04:34:57,734 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-14 04:34:57,734 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-14 04:34:57,734 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 04:34:57,735 main DEBUG OutputStream closed 2024-11-14 04:34:57,735 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 04:34:57,735 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 04:34:57,735 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-14 04:34:57,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 04:34:57,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 04:34:57,820 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 04:34:57,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 04:34:57,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 04:34:57,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 04:34:57,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 04:34:57,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 04:34:57,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 04:34:57,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 04:34:57,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 04:34:57,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 04:34:57,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 04:34:57,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 04:34:57,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 04:34:57,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 04:34:57,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 04:34:57,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 04:34:57,829 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 04:34:57,830 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-14 04:34:57,830 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 04:34:57,831 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-14T04:34:58,063 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e 2024-11-14 04:34:58,066 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 04:34:58,066 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T04:34:58,074 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-14T04:34:58,082 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-14T04:34:58,106 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T04:34:58,146 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T04:34:58,146 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T04:34:58,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T04:34:58,170 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708, deleteOnExit=true 2024-11-14T04:34:58,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T04:34:58,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/test.cache.data in system properties and HBase conf 2024-11-14T04:34:58,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T04:34:58,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/hadoop.log.dir in system properties and HBase conf 2024-11-14T04:34:58,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T04:34:58,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T04:34:58,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T04:34:58,244 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T04:34:58,332 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T04:34:58,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T04:34:58,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T04:34:58,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T04:34:58,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T04:34:58,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T04:34:58,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T04:34:58,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T04:34:58,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T04:34:58,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T04:34:58,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/nfs.dump.dir in system properties and HBase conf 2024-11-14T04:34:58,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/java.io.tmpdir in system properties and HBase conf 2024-11-14T04:34:58,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T04:34:58,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T04:34:58,346 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T04:34:59,421 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T04:34:59,486 INFO [Time-limited test {}] log.Log(170): Logging initialized @2511ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T04:34:59,552 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T04:34:59,613 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T04:34:59,636 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T04:34:59,636 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T04:34:59,638 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T04:34:59,659 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T04:34:59,662 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/hadoop.log.dir/,AVAILABLE} 2024-11-14T04:34:59,664 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T04:34:59,861 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/java.io.tmpdir/jetty-localhost-35039-hadoop-hdfs-3_4_1-tests_jar-_-any-16241091496019726088/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T04:34:59,867 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:35039} 2024-11-14T04:34:59,868 INFO [Time-limited test {}] server.Server(415): Started @2893ms 2024-11-14T04:35:00,452 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T04:35:00,459 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T04:35:00,460 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T04:35:00,461 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T04:35:00,461 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T04:35:00,461 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/hadoop.log.dir/,AVAILABLE} 2024-11-14T04:35:00,462 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T04:35:00,561 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/java.io.tmpdir/jetty-localhost-34363-hadoop-hdfs-3_4_1-tests_jar-_-any-18109534636813774817/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T04:35:00,562 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:34363} 2024-11-14T04:35:00,562 INFO [Time-limited test {}] server.Server(415): Started @3587ms 2024-11-14T04:35:00,608 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T04:35:00,720 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T04:35:00,728 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T04:35:00,730 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T04:35:00,730 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T04:35:00,731 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T04:35:00,733 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/hadoop.log.dir/,AVAILABLE} 2024-11-14T04:35:00,734 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T04:35:00,841 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/java.io.tmpdir/jetty-localhost-42067-hadoop-hdfs-3_4_1-tests_jar-_-any-3569773602955923422/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T04:35:00,841 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:42067} 2024-11-14T04:35:00,842 INFO [Time-limited test {}] server.Server(415): Started @3867ms 2024-11-14T04:35:00,844 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T04:35:00,883 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T04:35:00,889 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T04:35:00,891 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T04:35:00,892 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T04:35:00,892 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T04:35:00,893 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/hadoop.log.dir/,AVAILABLE} 2024-11-14T04:35:00,894 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T04:35:01,005 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/java.io.tmpdir/jetty-localhost-45129-hadoop-hdfs-3_4_1-tests_jar-_-any-1837217362430249840/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T04:35:01,005 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:45129} 2024-11-14T04:35:01,006 INFO [Time-limited test {}] server.Server(415): Started @4031ms 2024-11-14T04:35:01,007 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T04:35:02,464 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data2/current/BP-954332279-172.17.0.3-1731558898852/current, will proceed with Du for space computation calculation, 2024-11-14T04:35:02,464 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data3/current/BP-954332279-172.17.0.3-1731558898852/current, will proceed with Du for space computation calculation, 2024-11-14T04:35:02,464 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data1/current/BP-954332279-172.17.0.3-1731558898852/current, will proceed with Du for space computation calculation, 2024-11-14T04:35:02,464 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data4/current/BP-954332279-172.17.0.3-1731558898852/current, will proceed with Du for space computation calculation, 2024-11-14T04:35:02,489 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T04:35:02,490 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T04:35:02,532 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x99394bc37ae486a9 with lease ID 0x31ef442a4ef45d6b: Processing first storage report for DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b from datanode DatanodeRegistration(127.0.0.1:33533, datanodeUuid=337cb191-b596-474b-8076-6ace53fb5e5c, infoPort=46855, infoSecurePort=0, ipcPort=43583, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852) 2024-11-14T04:35:02,532 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data5/current/BP-954332279-172.17.0.3-1731558898852/current, will proceed with Du for space computation calculation, 2024-11-14T04:35:02,532 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data6/current/BP-954332279-172.17.0.3-1731558898852/current, will proceed with Du for space computation calculation, 2024-11-14T04:35:02,533 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x99394bc37ae486a9 with lease ID 0x31ef442a4ef45d6b: from storage DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b node DatanodeRegistration(127.0.0.1:33533, datanodeUuid=337cb191-b596-474b-8076-6ace53fb5e5c, infoPort=46855, infoSecurePort=0, ipcPort=43583, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T04:35:02,533 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3e34a445a9554b69 with lease ID 0x31ef442a4ef45d6c: Processing first storage report for DS-6d4a911c-8872-4483-b251-f501bed14cdd from datanode DatanodeRegistration(127.0.0.1:37377, datanodeUuid=6b000cc2-4eed-412d-9988-f74dc86d9c91, infoPort=44105, infoSecurePort=0, ipcPort=35041, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852) 2024-11-14T04:35:02,534 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3e34a445a9554b69 with lease ID 0x31ef442a4ef45d6c: from storage DS-6d4a911c-8872-4483-b251-f501bed14cdd node DatanodeRegistration(127.0.0.1:37377, datanodeUuid=6b000cc2-4eed-412d-9988-f74dc86d9c91, infoPort=44105, infoSecurePort=0, ipcPort=35041, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T04:35:02,534 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x99394bc37ae486a9 with lease ID 0x31ef442a4ef45d6b: Processing first storage report for DS-e10b851f-27ba-4875-9e1d-42d432393cb2 from datanode DatanodeRegistration(127.0.0.1:33533, datanodeUuid=337cb191-b596-474b-8076-6ace53fb5e5c, infoPort=46855, infoSecurePort=0, ipcPort=43583, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852) 2024-11-14T04:35:02,534 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x99394bc37ae486a9 with lease ID 0x31ef442a4ef45d6b: from storage DS-e10b851f-27ba-4875-9e1d-42d432393cb2 node DatanodeRegistration(127.0.0.1:33533, datanodeUuid=337cb191-b596-474b-8076-6ace53fb5e5c, infoPort=46855, infoSecurePort=0, ipcPort=43583, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T04:35:02,534 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3e34a445a9554b69 with lease ID 0x31ef442a4ef45d6c: Processing first storage report for DS-76c71046-c203-4dda-adda-d44ba974d2f9 from datanode DatanodeRegistration(127.0.0.1:37377, datanodeUuid=6b000cc2-4eed-412d-9988-f74dc86d9c91, infoPort=44105, infoSecurePort=0, ipcPort=35041, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852) 2024-11-14T04:35:02,534 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3e34a445a9554b69 with lease ID 0x31ef442a4ef45d6c: from storage DS-76c71046-c203-4dda-adda-d44ba974d2f9 node DatanodeRegistration(127.0.0.1:37377, datanodeUuid=6b000cc2-4eed-412d-9988-f74dc86d9c91, infoPort=44105, infoSecurePort=0, ipcPort=35041, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T04:35:02,552 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T04:35:02,557 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa7a1710f7e8ac26 with lease ID 0x31ef442a4ef45d6d: Processing first storage report for DS-70d64bb0-53e7-483c-982e-2c826e578c07 from datanode DatanodeRegistration(127.0.0.1:36191, datanodeUuid=69d0a3f2-cc3e-4761-920d-3b67f21672d1, infoPort=42129, infoSecurePort=0, ipcPort=41257, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852) 2024-11-14T04:35:02,557 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7a1710f7e8ac26 with lease ID 0x31ef442a4ef45d6d: from storage DS-70d64bb0-53e7-483c-982e-2c826e578c07 node DatanodeRegistration(127.0.0.1:36191, datanodeUuid=69d0a3f2-cc3e-4761-920d-3b67f21672d1, infoPort=42129, infoSecurePort=0, ipcPort=41257, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T04:35:02,557 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa7a1710f7e8ac26 with lease ID 0x31ef442a4ef45d6d: Processing first storage report for DS-7f61370f-051a-40bd-9e1a-dac6be6b4590 from datanode DatanodeRegistration(127.0.0.1:36191, datanodeUuid=69d0a3f2-cc3e-4761-920d-3b67f21672d1, infoPort=42129, infoSecurePort=0, ipcPort=41257, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852) 2024-11-14T04:35:02,558 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7a1710f7e8ac26 with lease ID 0x31ef442a4ef45d6d: from storage DS-7f61370f-051a-40bd-9e1a-dac6be6b4590 node DatanodeRegistration(127.0.0.1:36191, datanodeUuid=69d0a3f2-cc3e-4761-920d-3b67f21672d1, infoPort=42129, infoSecurePort=0, ipcPort=41257, storageInfo=lv=-57;cid=testClusterID;nsid=1976426055;c=1731558898852), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T04:35:02,612 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e 2024-11-14T04:35:02,671 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/zookeeper_0, clientPort=51455, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T04:35:02,679 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51455 2024-11-14T04:35:02,700 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:02,703 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:02,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741825_1001 (size=7) 2024-11-14T04:35:02,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741825_1001 (size=7) 2024-11-14T04:35:02,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741825_1001 (size=7) 2024-11-14T04:35:03,318 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc with version=8 2024-11-14T04:35:03,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/hbase-staging 2024-11-14T04:35:03,710 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b024da739ae8:0 server-side Connection retries=45 2024-11-14T04:35:03,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:03,719 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:03,723 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:35:03,723 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:03,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:35:03,842 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T04:35:03,892 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T04:35:03,900 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T04:35:03,903 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:35:03,926 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 98559 (auto-detected) 2024-11-14T04:35:03,926 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-14T04:35:03,943 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36617 2024-11-14T04:35:03,961 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36617 connecting to ZooKeeper ensemble=127.0.0.1:51455 2024-11-14T04:35:04,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:366170x0, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:35:04,080 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36617-0x10137a0072b0000 connected 2024-11-14T04:35:04,178 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:04,181 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:04,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:35:04,195 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc, hbase.cluster.distributed=false 2024-11-14T04:35:04,216 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:35:04,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36617 2024-11-14T04:35:04,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36617 2024-11-14T04:35:04,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36617 2024-11-14T04:35:04,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36617 2024-11-14T04:35:04,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36617 2024-11-14T04:35:04,317 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b024da739ae8:0 server-side Connection retries=45 2024-11-14T04:35:04,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:04,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:04,319 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:35:04,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:04,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:35:04,322 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T04:35:04,325 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:35:04,326 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43439 2024-11-14T04:35:04,328 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43439 connecting to ZooKeeper ensemble=127.0.0.1:51455 2024-11-14T04:35:04,329 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:04,333 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:04,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434390x0, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:35:04,346 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:434390x0, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:35:04,346 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43439-0x10137a0072b0001 connected 2024-11-14T04:35:04,351 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T04:35:04,362 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T04:35:04,365 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T04:35:04,371 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:35:04,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43439 2024-11-14T04:35:04,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43439 2024-11-14T04:35:04,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43439 2024-11-14T04:35:04,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43439 2024-11-14T04:35:04,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43439 2024-11-14T04:35:04,394 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b024da739ae8:0 server-side Connection retries=45 2024-11-14T04:35:04,394 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:04,394 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:04,395 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:35:04,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:04,395 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:35:04,395 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T04:35:04,396 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:35:04,397 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42805 2024-11-14T04:35:04,398 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42805 connecting to ZooKeeper ensemble=127.0.0.1:51455 2024-11-14T04:35:04,399 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:04,403 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:04,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428050x0, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:35:04,420 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:428050x0, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:35:04,420 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42805-0x10137a0072b0002 connected 2024-11-14T04:35:04,421 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T04:35:04,422 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T04:35:04,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T04:35:04,425 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:35:04,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42805 2024-11-14T04:35:04,427 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42805 2024-11-14T04:35:04,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42805 2024-11-14T04:35:04,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42805 2024-11-14T04:35:04,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42805 2024-11-14T04:35:04,446 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b024da739ae8:0 server-side Connection retries=45 2024-11-14T04:35:04,447 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:04,447 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:04,447 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T04:35:04,447 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T04:35:04,447 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T04:35:04,448 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T04:35:04,448 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T04:35:04,449 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38767 2024-11-14T04:35:04,451 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38767 connecting to ZooKeeper ensemble=127.0.0.1:51455 2024-11-14T04:35:04,452 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:04,454 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:04,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387670x0, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T04:35:04,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387670x0, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:35:04,471 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38767-0x10137a0072b0003 connected 2024-11-14T04:35:04,471 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T04:35:04,472 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T04:35:04,473 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T04:35:04,475 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T04:35:04,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38767 2024-11-14T04:35:04,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38767 2024-11-14T04:35:04,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38767 2024-11-14T04:35:04,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38767 2024-11-14T04:35:04,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38767 2024-11-14T04:35:04,492 DEBUG [M:0;b024da739ae8:36617 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b024da739ae8:36617 2024-11-14T04:35:04,493 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b024da739ae8,36617,1731558903563 2024-11-14T04:35:04,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:35:04,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:35:04,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:35:04,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:35:04,506 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b024da739ae8,36617,1731558903563 2024-11-14T04:35:04,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T04:35:04,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T04:35:04,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:04,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T04:35:04,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:04,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:04,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:04,536 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T04:35:04,538 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b024da739ae8,36617,1731558903563 from backup master directory 2024-11-14T04:35:04,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b024da739ae8,36617,1731558903563 2024-11-14T04:35:04,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:35:04,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:35:04,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:35:04,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T04:35:04,546 WARN [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:35:04,547 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b024da739ae8,36617,1731558903563 2024-11-14T04:35:04,549 INFO [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T04:35:04,551 INFO [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T04:35:04,604 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/hbase.id] with ID: ead3ed9a-fa97-4c97-9e40-e4af243b7358 2024-11-14T04:35:04,604 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/.tmp/hbase.id 2024-11-14T04:35:04,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741826_1002 (size=42) 2024-11-14T04:35:04,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741826_1002 (size=42) 2024-11-14T04:35:04,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741826_1002 (size=42) 2024-11-14T04:35:04,619 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/.tmp/hbase.id]:[hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/hbase.id] 2024-11-14T04:35:04,664 INFO [master/b024da739ae8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T04:35:04,669 INFO [master/b024da739ae8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T04:35:04,686 INFO [master/b024da739ae8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-14T04:35:04,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:04,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:04,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:04,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:04,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741827_1003 (size=196) 2024-11-14T04:35:04,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741827_1003 (size=196) 2024-11-14T04:35:04,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741827_1003 (size=196) 2024-11-14T04:35:04,726 INFO [master/b024da739ae8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T04:35:04,728 INFO [master/b024da739ae8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T04:35:04,732 INFO [master/b024da739ae8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:04,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741828_1004 (size=1189) 2024-11-14T04:35:04,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741828_1004 (size=1189) 2024-11-14T04:35:04,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741828_1004 (size=1189) 2024-11-14T04:35:04,780 INFO [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store 2024-11-14T04:35:04,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741829_1005 (size=34) 2024-11-14T04:35:04,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741829_1005 (size=34) 2024-11-14T04:35:04,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741829_1005 (size=34) 2024-11-14T04:35:04,807 INFO [master/b024da739ae8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T04:35:04,810 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:04,811 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T04:35:04,812 INFO [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:35:04,812 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:35:04,814 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T04:35:04,814 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:35:04,814 INFO [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:35:04,815 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731558904811Disabling compacts and flushes for region at 1731558904811Disabling writes for close at 1731558904814 (+3 ms)Writing region close event to WAL at 1731558904814Closed at 1731558904814 2024-11-14T04:35:04,817 WARN [master/b024da739ae8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/.initializing 2024-11-14T04:35:04,818 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/WALs/b024da739ae8,36617,1731558903563 2024-11-14T04:35:04,825 INFO [master/b024da739ae8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:04,839 INFO [master/b024da739ae8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b024da739ae8%2C36617%2C1731558903563, suffix=, logDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/WALs/b024da739ae8,36617,1731558903563, archiveDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/oldWALs, maxLogs=10 2024-11-14T04:35:04,866 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/WALs/b024da739ae8,36617,1731558903563/b024da739ae8%2C36617%2C1731558903563.1731558904843, exclude list is [], retry=0 2024-11-14T04:35:04,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:04,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:04,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:04,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:04,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T04:35:04,921 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-14T04:35:04,976 INFO [master/b024da739ae8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/WALs/b024da739ae8,36617,1731558903563/b024da739ae8%2C36617%2C1731558903563.1731558904843 2024-11-14T04:35:04,977 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:46855:46855)] 2024-11-14T04:35:04,978 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:04,978 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:04,982 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:04,982 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,037 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T04:35:05,040 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:05,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:35:05,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T04:35:05,046 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:05,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:05,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,051 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T04:35:05,051 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:05,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:05,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T04:35:05,056 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:05,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:05,058 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,061 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,062 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,067 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,068 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,071 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T04:35:05,074 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T04:35:05,078 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:05,079 INFO [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65005635, jitterRate=-0.03134055435657501}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T04:35:05,087 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731558904994Initializing all the Stores at 1731558904996 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731558904996Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558904997 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558904997Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558904997Cleaning up temporary data from old regions at 1731558905068 (+71 ms)Region opened successfully at 1731558905087 (+19 ms) 2024-11-14T04:35:05,088 INFO [master/b024da739ae8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T04:35:05,122 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e994fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b024da739ae8/172.17.0.3:0 2024-11-14T04:35:05,147 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T04:35:05,156 INFO [master/b024da739ae8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T04:35:05,157 INFO [master/b024da739ae8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T04:35:05,159 INFO [master/b024da739ae8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T04:35:05,160 INFO [master/b024da739ae8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T04:35:05,164 INFO [master/b024da739ae8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T04:35:05,164 INFO [master/b024da739ae8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T04:35:05,187 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T04:35:05,195 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T04:35:05,208 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T04:35:05,211 INFO [master/b024da739ae8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T04:35:05,212 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T04:35:05,219 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T04:35:05,221 INFO [master/b024da739ae8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T04:35:05,224 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T04:35:05,229 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T04:35:05,230 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T04:35:05,240 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T04:35:05,259 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T04:35:05,271 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T04:35:05,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T04:35:05,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T04:35:05,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T04:35:05,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T04:35:05,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,287 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b024da739ae8,36617,1731558903563, sessionid=0x10137a0072b0000, setting cluster-up flag (Was=false) 2024-11-14T04:35:05,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,346 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T04:35:05,351 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b024da739ae8,36617,1731558903563 2024-11-14T04:35:05,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:05,408 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T04:35:05,411 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b024da739ae8,36617,1731558903563 2024-11-14T04:35:05,423 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T04:35:05,481 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(746): ClusterId : ead3ed9a-fa97-4c97-9e40-e4af243b7358 2024-11-14T04:35:05,481 INFO [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(746): ClusterId : ead3ed9a-fa97-4c97-9e40-e4af243b7358 2024-11-14T04:35:05,481 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(746): ClusterId : ead3ed9a-fa97-4c97-9e40-e4af243b7358 2024-11-14T04:35:05,484 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T04:35:05,484 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T04:35:05,484 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T04:35:05,488 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T04:35:05,495 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T04:35:05,495 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T04:35:05,495 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T04:35:05,495 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T04:35:05,496 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T04:35:05,496 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T04:35:05,498 INFO [master/b024da739ae8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T04:35:05,504 INFO [master/b024da739ae8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T04:35:05,510 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b024da739ae8,36617,1731558903563 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T04:35:05,514 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T04:35:05,514 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T04:35:05,514 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T04:35:05,515 DEBUG [RS:1;b024da739ae8:42805 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1650d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b024da739ae8/172.17.0.3:0 2024-11-14T04:35:05,515 DEBUG [RS:0;b024da739ae8:43439 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa05ae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b024da739ae8/172.17.0.3:0 2024-11-14T04:35:05,515 DEBUG [RS:2;b024da739ae8:38767 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3afb2770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b024da739ae8/172.17.0.3:0 2024-11-14T04:35:05,518 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b024da739ae8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:35:05,519 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b024da739ae8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:35:05,519 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b024da739ae8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:35:05,519 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b024da739ae8:0, corePoolSize=5, maxPoolSize=5 2024-11-14T04:35:05,519 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b024da739ae8:0, corePoolSize=10, maxPoolSize=10 2024-11-14T04:35:05,519 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,520 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b024da739ae8:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:35:05,520 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,528 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T04:35:05,529 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T04:35:05,531 DEBUG [RS:0;b024da739ae8:43439 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b024da739ae8:43439 2024-11-14T04:35:05,533 DEBUG [RS:2;b024da739ae8:38767 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;b024da739ae8:38767 2024-11-14T04:35:05,534 DEBUG [RS:1;b024da739ae8:42805 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b024da739ae8:42805 2024-11-14T04:35:05,535 INFO [RS:1;b024da739ae8:42805 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T04:35:05,535 INFO [RS:0;b024da739ae8:43439 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T04:35:05,535 INFO [RS:2;b024da739ae8:38767 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T04:35:05,536 INFO [RS:1;b024da739ae8:42805 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T04:35:05,536 INFO [RS:2;b024da739ae8:38767 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T04:35:05,536 INFO [RS:0;b024da739ae8:43439 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T04:35:05,536 DEBUG [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T04:35:05,536 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:05,536 DEBUG [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T04:35:05,536 DEBUG [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T04:35:05,536 INFO [master/b024da739ae8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731558935536 2024-11-14T04:35:05,536 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T04:35:05,538 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T04:35:05,538 INFO [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(2659): reportForDuty to master=b024da739ae8,36617,1731558903563 with port=38767, startcode=1731558904446 2024-11-14T04:35:05,538 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(2659): reportForDuty to master=b024da739ae8,36617,1731558903563 with port=42805, startcode=1731558904393 2024-11-14T04:35:05,538 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(2659): reportForDuty to master=b024da739ae8,36617,1731558903563 with port=43439, startcode=1731558904285 2024-11-14T04:35:05,539 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T04:35:05,543 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T04:35:05,544 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T04:35:05,544 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T04:35:05,544 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T04:35:05,548 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,552 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T04:35:05,552 DEBUG [RS:1;b024da739ae8:42805 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T04:35:05,552 DEBUG [RS:2;b024da739ae8:38767 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T04:35:05,552 DEBUG [RS:0;b024da739ae8:43439 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T04:35:05,553 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T04:35:05,554 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T04:35:05,561 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T04:35:05,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741831_1007 (size=1321) 2024-11-14T04:35:05,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741831_1007 (size=1321) 2024-11-14T04:35:05,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741831_1007 (size=1321) 2024-11-14T04:35:05,568 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T04:35:05,568 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc 2024-11-14T04:35:05,592 INFO [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T04:35:05,595 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b024da739ae8:0:becomeActiveMaster-HFileCleaner.large.0-1731558905594,5,FailOnTimeoutGroup] 2024-11-14T04:35:05,596 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b024da739ae8:0:becomeActiveMaster-HFileCleaner.small.0-1731558905595,5,FailOnTimeoutGroup] 2024-11-14T04:35:05,596 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,597 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T04:35:05,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741832_1008 (size=32) 2024-11-14T04:35:05,598 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,599 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741832_1008 (size=32) 2024-11-14T04:35:05,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741832_1008 (size=32) 2024-11-14T04:35:05,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:05,603 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59701, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T04:35:05,603 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40047, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T04:35:05,603 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43177, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T04:35:05,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T04:35:05,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T04:35:05,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:05,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:35:05,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T04:35:05,611 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36617 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b024da739ae8,42805,1731558904393 2024-11-14T04:35:05,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T04:35:05,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:05,614 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36617 {}] master.ServerManager(517): Registering regionserver=b024da739ae8,42805,1731558904393 2024-11-14T04:35:05,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:35:05,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T04:35:05,618 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T04:35:05,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:05,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:35:05,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T04:35:05,624 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T04:35:05,624 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:05,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:35:05,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T04:35:05,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740 2024-11-14T04:35:05,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740 2024-11-14T04:35:05,629 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36617 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b024da739ae8,38767,1731558904446 2024-11-14T04:35:05,629 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36617 {}] master.ServerManager(517): Registering regionserver=b024da739ae8,38767,1731558904446 2024-11-14T04:35:05,632 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T04:35:05,632 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T04:35:05,633 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T04:35:05,633 DEBUG [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc 2024-11-14T04:35:05,633 DEBUG [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41731 2024-11-14T04:35:05,633 DEBUG [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T04:35:05,635 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36617 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b024da739ae8,43439,1731558904285 2024-11-14T04:35:05,635 DEBUG [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc 2024-11-14T04:35:05,635 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36617 {}] master.ServerManager(517): Registering regionserver=b024da739ae8,43439,1731558904285 2024-11-14T04:35:05,635 DEBUG [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41731 2024-11-14T04:35:05,635 DEBUG [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T04:35:05,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T04:35:05,640 DEBUG [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc 2024-11-14T04:35:05,640 DEBUG [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41731 2024-11-14T04:35:05,640 DEBUG [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T04:35:05,640 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:05,641 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66714382, jitterRate=-0.005878239870071411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T04:35:05,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731558905603Initializing all the Stores at 1731558905605 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731558905606 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731558905606Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558905606Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731558905606Cleaning up temporary data from old regions at 1731558905632 (+26 ms)Region opened successfully at 1731558905645 (+13 ms) 2024-11-14T04:35:05,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T04:35:05,645 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T04:35:05,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T04:35:05,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T04:35:05,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T04:35:05,647 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T04:35:05,647 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731558905645Disabling compacts and flushes for region at 1731558905645Disabling writes for close at 1731558905645Writing region close event to WAL at 1731558905647 (+2 ms)Closed at 1731558905647 2024-11-14T04:35:05,650 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T04:35:05,650 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T04:35:05,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T04:35:05,664 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T04:35:05,666 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T04:35:05,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:35:05,693 DEBUG [RS:2;b024da739ae8:38767 {}] zookeeper.ZKUtil(111): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b024da739ae8,38767,1731558904446 2024-11-14T04:35:05,693 DEBUG [RS:1;b024da739ae8:42805 {}] zookeeper.ZKUtil(111): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b024da739ae8,42805,1731558904393 2024-11-14T04:35:05,693 WARN [RS:1;b024da739ae8:42805 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:35:05,693 WARN [RS:2;b024da739ae8:38767 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:35:05,693 INFO [RS:2;b024da739ae8:38767 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:05,693 INFO [RS:1;b024da739ae8:42805 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:05,694 DEBUG [RS:0;b024da739ae8:43439 {}] zookeeper.ZKUtil(111): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b024da739ae8,43439,1731558904285 2024-11-14T04:35:05,694 DEBUG [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,42805,1731558904393 2024-11-14T04:35:05,694 DEBUG [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,38767,1731558904446 2024-11-14T04:35:05,694 WARN [RS:0;b024da739ae8:43439 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T04:35:05,694 INFO [RS:0;b024da739ae8:43439 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:05,694 DEBUG [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285 2024-11-14T04:35:05,695 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b024da739ae8,42805,1731558904393] 2024-11-14T04:35:05,695 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b024da739ae8,43439,1731558904285] 2024-11-14T04:35:05,695 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b024da739ae8,38767,1731558904446] 2024-11-14T04:35:05,721 INFO [RS:0;b024da739ae8:43439 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T04:35:05,721 INFO [RS:1;b024da739ae8:42805 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T04:35:05,721 INFO [RS:2;b024da739ae8:38767 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T04:35:05,738 INFO [RS:1;b024da739ae8:42805 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T04:35:05,738 INFO [RS:2;b024da739ae8:38767 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T04:35:05,738 INFO [RS:0;b024da739ae8:43439 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T04:35:05,743 INFO [RS:2;b024da739ae8:38767 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T04:35:05,743 INFO [RS:0;b024da739ae8:43439 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T04:35:05,743 INFO [RS:1;b024da739ae8:42805 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T04:35:05,743 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,744 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,744 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,745 INFO [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T04:35:05,745 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T04:35:05,745 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T04:35:05,750 INFO [RS:0;b024da739ae8:43439 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T04:35:05,750 INFO [RS:2;b024da739ae8:38767 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T04:35:05,750 INFO [RS:1;b024da739ae8:42805 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T04:35:05,752 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,752 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,752 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,752 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,752 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,752 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,752 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,752 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,752 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,752 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,752 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,752 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,752 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,752 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:35:05,753 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:35:05,753 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0, corePoolSize=2, maxPoolSize=2 2024-11-14T04:35:05,753 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b024da739ae8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:35:05,753 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b024da739ae8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:35:05,753 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,753 DEBUG [RS:0;b024da739ae8:43439 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b024da739ae8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:35:05,753 DEBUG [RS:2;b024da739ae8:38767 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b024da739ae8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:35:05,754 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,754 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,754 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b024da739ae8:0, corePoolSize=1, maxPoolSize=1 2024-11-14T04:35:05,754 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b024da739ae8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:35:05,754 DEBUG [RS:1;b024da739ae8:42805 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b024da739ae8:0, corePoolSize=3, maxPoolSize=3 2024-11-14T04:35:05,757 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,757 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,757 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,757 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,757 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,758 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,43439,1731558904285-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:35:05,760 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,760 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,760 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,760 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,761 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,761 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,38767,1731558904446-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:35:05,761 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,761 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,761 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,761 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,761 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,761 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,42805,1731558904393-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:35:05,778 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T04:35:05,780 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,43439,1731558904285-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,780 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,780 INFO [RS:0;b024da739ae8:43439 {}] regionserver.Replication(171): b024da739ae8,43439,1731558904285 started 2024-11-14T04:35:05,783 INFO [RS:2;b024da739ae8:38767 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T04:35:05,783 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,38767,1731558904446-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,783 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T04:35:05,783 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,783 INFO [RS:2;b024da739ae8:38767 {}] regionserver.Replication(171): b024da739ae8,38767,1731558904446 started 2024-11-14T04:35:05,783 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,42805,1731558904393-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,783 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,783 INFO [RS:1;b024da739ae8:42805 {}] regionserver.Replication(171): b024da739ae8,42805,1731558904393 started 2024-11-14T04:35:05,797 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,798 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(1482): Serving as b024da739ae8,43439,1731558904285, RpcServer on b024da739ae8/172.17.0.3:43439, sessionid=0x10137a0072b0001 2024-11-14T04:35:05,799 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T04:35:05,799 DEBUG [RS:0;b024da739ae8:43439 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b024da739ae8,43439,1731558904285 2024-11-14T04:35:05,799 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b024da739ae8,43439,1731558904285' 2024-11-14T04:35:05,799 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T04:35:05,800 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T04:35:05,801 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T04:35:05,801 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T04:35:05,801 DEBUG [RS:0;b024da739ae8:43439 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b024da739ae8,43439,1731558904285 2024-11-14T04:35:05,801 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b024da739ae8,43439,1731558904285' 2024-11-14T04:35:05,801 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T04:35:05,802 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T04:35:05,802 DEBUG [RS:0;b024da739ae8:43439 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T04:35:05,802 INFO [RS:0;b024da739ae8:43439 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T04:35:05,802 INFO [RS:0;b024da739ae8:43439 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T04:35:05,806 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,806 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:05,806 INFO [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(1482): Serving as b024da739ae8,38767,1731558904446, RpcServer on b024da739ae8/172.17.0.3:38767, sessionid=0x10137a0072b0003 2024-11-14T04:35:05,806 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(1482): Serving as b024da739ae8,42805,1731558904393, RpcServer on b024da739ae8/172.17.0.3:42805, sessionid=0x10137a0072b0002 2024-11-14T04:35:05,806 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T04:35:05,806 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T04:35:05,806 DEBUG [RS:2;b024da739ae8:38767 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b024da739ae8,38767,1731558904446 2024-11-14T04:35:05,806 DEBUG [RS:1;b024da739ae8:42805 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b024da739ae8,42805,1731558904393 2024-11-14T04:35:05,806 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b024da739ae8,42805,1731558904393' 2024-11-14T04:35:05,806 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b024da739ae8,38767,1731558904446' 2024-11-14T04:35:05,806 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T04:35:05,806 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T04:35:05,807 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T04:35:05,807 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T04:35:05,808 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T04:35:05,808 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T04:35:05,808 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T04:35:05,808 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T04:35:05,808 DEBUG [RS:2;b024da739ae8:38767 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b024da739ae8,38767,1731558904446 2024-11-14T04:35:05,808 DEBUG [RS:1;b024da739ae8:42805 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b024da739ae8,42805,1731558904393 2024-11-14T04:35:05,808 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b024da739ae8,38767,1731558904446' 2024-11-14T04:35:05,808 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b024da739ae8,42805,1731558904393' 2024-11-14T04:35:05,808 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T04:35:05,808 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T04:35:05,809 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T04:35:05,809 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T04:35:05,809 DEBUG [RS:1;b024da739ae8:42805 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T04:35:05,809 DEBUG [RS:2;b024da739ae8:38767 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T04:35:05,809 INFO [RS:1;b024da739ae8:42805 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T04:35:05,809 INFO [RS:2;b024da739ae8:38767 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T04:35:05,809 INFO [RS:1;b024da739ae8:42805 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T04:35:05,809 INFO [RS:2;b024da739ae8:38767 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T04:35:05,817 WARN [b024da739ae8:36617 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T04:35:05,907 INFO [RS:0;b024da739ae8:43439 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:05,910 INFO [RS:1;b024da739ae8:42805 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:05,910 INFO [RS:2;b024da739ae8:38767 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:05,910 INFO [RS:0;b024da739ae8:43439 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b024da739ae8%2C43439%2C1731558904285, suffix=, logDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285, archiveDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/oldWALs, maxLogs=32 2024-11-14T04:35:05,913 INFO [RS:1;b024da739ae8:42805 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b024da739ae8%2C42805%2C1731558904393, suffix=, logDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,42805,1731558904393, archiveDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/oldWALs, maxLogs=32 2024-11-14T04:35:05,913 INFO [RS:2;b024da739ae8:38767 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b024da739ae8%2C38767%2C1731558904446, suffix=, logDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,38767,1731558904446, archiveDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/oldWALs, maxLogs=32 2024-11-14T04:35:05,930 DEBUG [RS:1;b024da739ae8:42805 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,42805,1731558904393/b024da739ae8%2C42805%2C1731558904393.1731558905917, exclude list is [], retry=0 2024-11-14T04:35:05,931 DEBUG [RS:0;b024da739ae8:43439 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285/b024da739ae8%2C43439%2C1731558904285.1731558905913, exclude list is [], retry=0 2024-11-14T04:35:05,936 DEBUG [RS:2;b024da739ae8:38767 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,38767,1731558904446/b024da739ae8%2C38767%2C1731558904446.1731558905917, exclude list is [], retry=0 2024-11-14T04:35:05,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:05,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:05,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:05,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:05,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:05,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:05,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:05,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:05,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:05,984 INFO [RS:1;b024da739ae8:42805 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,42805,1731558904393/b024da739ae8%2C42805%2C1731558904393.1731558905917 2024-11-14T04:35:05,985 DEBUG [RS:1;b024da739ae8:42805 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-14T04:35:05,991 INFO [RS:0;b024da739ae8:43439 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285/b024da739ae8%2C43439%2C1731558904285.1731558905913 2024-11-14T04:35:05,992 INFO [RS:2;b024da739ae8:38767 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,38767,1731558904446/b024da739ae8%2C38767%2C1731558904446.1731558905917 2024-11-14T04:35:05,992 DEBUG [RS:0;b024da739ae8:43439 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855)] 2024-11-14T04:35:05,993 DEBUG [RS:2;b024da739ae8:38767 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-14T04:35:06,073 DEBUG [b024da739ae8:36617 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-14T04:35:06,084 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(204): Hosts are {b024da739ae8=0} racks are {/default-rack=0} 2024-11-14T04:35:06,090 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T04:35:06,090 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T04:35:06,090 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T04:35:06,090 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T04:35:06,090 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T04:35:06,090 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T04:35:06,090 INFO [b024da739ae8:36617 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T04:35:06,090 INFO [b024da739ae8:36617 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T04:35:06,090 INFO [b024da739ae8:36617 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T04:35:06,090 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T04:35:06,096 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b024da739ae8,42805,1731558904393 2024-11-14T04:35:06,101 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b024da739ae8,42805,1731558904393, state=OPENING 2024-11-14T04:35:06,156 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T04:35:06,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:06,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:06,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:06,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:06,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:35:06,169 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:35:06,170 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:35:06,170 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:35:06,173 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T04:35:06,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b024da739ae8,42805,1731558904393}] 2024-11-14T04:35:06,348 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T04:35:06,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41997, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T04:35:06,364 INFO [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T04:35:06,365 INFO [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:06,365 INFO [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T04:35:06,369 INFO [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b024da739ae8%2C42805%2C1731558904393.meta, suffix=.meta, logDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,42805,1731558904393, archiveDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/oldWALs, maxLogs=32 2024-11-14T04:35:06,385 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,42805,1731558904393/b024da739ae8%2C42805%2C1731558904393.meta.1731558906371.meta, exclude list is [], retry=0 2024-11-14T04:35:06,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:06,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:06,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:06,416 INFO [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,42805,1731558904393/b024da739ae8%2C42805%2C1731558904393.meta.1731558906371.meta 2024-11-14T04:35:06,417 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:06,417 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:06,419 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T04:35:06,421 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T04:35:06,425 INFO [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T04:35:06,430 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T04:35:06,430 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:06,431 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T04:35:06,431 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T04:35:06,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T04:35:06,436 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T04:35:06,436 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:06,437 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:35:06,437 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T04:35:06,439 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T04:35:06,439 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:06,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:35:06,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T04:35:06,441 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T04:35:06,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:06,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:35:06,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T04:35:06,444 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T04:35:06,444 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:06,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T04:35:06,445 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T04:35:06,447 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740 2024-11-14T04:35:06,450 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740 2024-11-14T04:35:06,453 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T04:35:06,453 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T04:35:06,454 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T04:35:06,457 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T04:35:06,459 INFO [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65979155, jitterRate=-0.016833975911140442}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T04:35:06,459 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T04:35:06,461 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731558906431Writing region info on filesystem at 1731558906432 (+1 ms)Initializing all the Stores at 1731558906433 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731558906433Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731558906434 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558906434Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731558906434Cleaning up temporary data from old regions at 1731558906453 (+19 ms)Running coprocessor post-open hooks at 1731558906460 (+7 ms)Region opened successfully at 1731558906461 (+1 ms) 2024-11-14T04:35:06,467 INFO [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731558906342 2024-11-14T04:35:06,477 DEBUG [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T04:35:06,478 INFO [RS_OPEN_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T04:35:06,480 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b024da739ae8,42805,1731558904393 2024-11-14T04:35:06,482 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b024da739ae8,42805,1731558904393, state=OPEN 2024-11-14T04:35:06,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T04:35:06,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T04:35:06,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T04:35:06,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T04:35:06,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:35:06,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:35:06,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:35:06,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T04:35:06,565 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b024da739ae8,42805,1731558904393 2024-11-14T04:35:06,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T04:35:06,573 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b024da739ae8,42805,1731558904393 in 390 msec 2024-11-14T04:35:06,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T04:35:06,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 919 msec 2024-11-14T04:35:06,581 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T04:35:06,582 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T04:35:06,599 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T04:35:06,600 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b024da739ae8,42805,1731558904393, seqNum=-1] 2024-11-14T04:35:06,619 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T04:35:06,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44825, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T04:35:06,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1900 sec 2024-11-14T04:35:06,642 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731558906642, completionTime=-1 2024-11-14T04:35:06,644 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-14T04:35:06,644 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T04:35:06,692 INFO [master/b024da739ae8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-14T04:35:06,692 INFO [master/b024da739ae8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731558966692 2024-11-14T04:35:06,692 INFO [master/b024da739ae8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731559026692 2024-11-14T04:35:06,692 INFO [master/b024da739ae8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 47 msec 2024-11-14T04:35:06,694 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-14T04:35:06,701 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,36617,1731558903563-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:06,701 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,36617,1731558903563-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:06,701 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,36617,1731558903563-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:06,707 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b024da739ae8:36617, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:06,707 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:06,708 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:06,716 DEBUG [master/b024da739ae8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T04:35:06,736 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.189sec 2024-11-14T04:35:06,737 INFO [master/b024da739ae8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T04:35:06,738 INFO [master/b024da739ae8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T04:35:06,739 INFO [master/b024da739ae8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T04:35:06,740 INFO [master/b024da739ae8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T04:35:06,740 INFO [master/b024da739ae8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T04:35:06,741 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,36617,1731558903563-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T04:35:06,741 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,36617,1731558903563-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T04:35:06,746 DEBUG [master/b024da739ae8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T04:35:06,747 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T04:35:06,747 INFO [master/b024da739ae8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b024da739ae8,36617,1731558903563-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:06,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600d4a87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T04:35:06,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b024da739ae8,36617,-1 for getting cluster id 2024-11-14T04:35:06,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T04:35:06,806 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ead3ed9a-fa97-4c97-9e40-e4af243b7358' 2024-11-14T04:35:06,808 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T04:35:06,809 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ead3ed9a-fa97-4c97-9e40-e4af243b7358" 2024-11-14T04:35:06,809 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aa4535d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T04:35:06,809 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b024da739ae8,36617,-1] 2024-11-14T04:35:06,813 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T04:35:06,815 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:35:06,816 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T04:35:06,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be8dbe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T04:35:06,819 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T04:35:06,825 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b024da739ae8,42805,1731558904393, seqNum=-1] 2024-11-14T04:35:06,826 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T04:35:06,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T04:35:06,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b024da739ae8,36617,1731558903563 2024-11-14T04:35:06,848 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:41731/hbase 2024-11-14T04:35:06,863 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=464, ProcessCount=11, AvailableMemoryMB=7123 2024-11-14T04:35:06,888 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:06,892 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:06,893 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:06,898 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-68006507, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/hregion-68006507, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:06,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-68006507/hregion-68006507.1731558906899, exclude list is [], retry=0 2024-11-14T04:35:06,920 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:06,920 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:06,921 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:06,949 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-68006507/hregion-68006507.1731558906899 2024-11-14T04:35:06,949 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:06,950 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 048f68c571159453a72905c3b4cf6207, NAME => 'testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/hbase 2024-11-14T04:35:06,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741838_1014 (size=64) 2024-11-14T04:35:06,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741838_1014 (size=64) 2024-11-14T04:35:06,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741838_1014 (size=64) 2024-11-14T04:35:06,962 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:06,964 INFO [StoreOpener-048f68c571159453a72905c3b4cf6207-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:06,966 INFO [StoreOpener-048f68c571159453a72905c3b4cf6207-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 048f68c571159453a72905c3b4cf6207 columnFamilyName a 2024-11-14T04:35:06,966 DEBUG [StoreOpener-048f68c571159453a72905c3b4cf6207-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:06,967 INFO [StoreOpener-048f68c571159453a72905c3b4cf6207-1 {}] regionserver.HStore(327): Store=048f68c571159453a72905c3b4cf6207/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:06,967 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:06,969 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:06,969 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:06,970 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:06,970 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:06,973 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:06,976 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:06,977 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 048f68c571159453a72905c3b4cf6207; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67038143, jitterRate=-0.0010538250207901}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:35:06,978 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 048f68c571159453a72905c3b4cf6207: Writing region info on filesystem at 1731558906962Initializing all the Stores at 1731558906964 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558906964Cleaning up temporary data from old regions at 1731558906970 (+6 ms)Region opened successfully at 1731558906978 (+8 ms) 2024-11-14T04:35:06,978 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 048f68c571159453a72905c3b4cf6207, disabling compactions & flushes 2024-11-14T04:35:06,978 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207. 2024-11-14T04:35:06,978 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207. 2024-11-14T04:35:06,978 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207. after waiting 0 ms 2024-11-14T04:35:06,978 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207. 2024-11-14T04:35:06,979 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207. 2024-11-14T04:35:06,979 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 048f68c571159453a72905c3b4cf6207: Waiting for close lock at 1731558906978Disabling compacts and flushes for region at 1731558906978Disabling writes for close at 1731558906978Writing region close event to WAL at 1731558906979 (+1 ms)Closed at 1731558906979 2024-11-14T04:35:06,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741837_1013 (size=175) 2024-11-14T04:35:06,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741837_1013 (size=175) 2024-11-14T04:35:06,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741837_1013 (size=175) 2024-11-14T04:35:06,991 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:06,992 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-68006507:(num 1731558906899) 2024-11-14T04:35:06,994 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T04:35:07,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741839_1015 (size=469) 2024-11-14T04:35:07,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741839_1015 (size=469) 2024-11-14T04:35:07,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741839_1015 (size=469) 2024-11-14T04:35:07,037 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T04:35:07,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741840_1016 (size=322) 2024-11-14T04:35:07,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741840_1016 (size=322) 2024-11-14T04:35:07,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741840_1016 (size=322) 2024-11-14T04:35:07,100 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-1, size=469 (469bytes) 2024-11-14T04:35:07,100 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-14T04:35:07,101 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-14T04:35:07,101 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-1 2024-11-14T04:35:07,105 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-1 after 2ms 2024-11-14T04:35:07,134 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:07,135 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-1 took 35ms 2024-11-14T04:35:07,140 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-1 so closing down 2024-11-14T04:35:07,140 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:07,143 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-14T04:35:07,168 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000001-wal-1.temp 2024-11-14T04:35:07,169 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:07,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741841_1017 (size=469) 2024-11-14T04:35:07,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741841_1017 (size=469) 2024-11-14T04:35:07,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741841_1017 (size=469) 2024-11-14T04:35:07,182 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:07,185 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000002 2024-11-14T04:35:07,190 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-14T04:35:07,190 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-1, journal: Splitting hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-1, size=469 (469bytes) at 1731558907100Finishing writing output for hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-1 so closing down at 1731558907140 (+40 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000001-wal-1.temp at 1731558907168 (+28 ms)3 split writer threads finished at 1731558907169 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731558907182 (+13 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000002 at 1731558907185 (+3 ms)Processed 2 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731558907190 (+5 ms) 2024-11-14T04:35:07,206 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-2, size=322 (322bytes) 2024-11-14T04:35:07,206 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-2 2024-11-14T04:35:07,207 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-2 after 1ms 2024-11-14T04:35:07,232 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:07,233 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-2 took 27ms 2024-11-14T04:35:07,235 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-2 so closing down 2024-11-14T04:35:07,235 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:07,238 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-14T04:35:07,265 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000002-wal-2.temp 2024-11-14T04:35:07,266 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:07,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741842_1018 (size=322) 2024-11-14T04:35:07,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741842_1018 (size=322) 2024-11-14T04:35:07,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741842_1018 (size=322) 2024-11-14T04:35:07,277 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:07,304 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:07,307 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-14T04:35:07,308 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-14T04:35:07,308 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-2, journal: Splitting hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-2, size=322 (322bytes) at 1731558907206Finishing writing output for hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-2 so closing down at 1731558907235 (+29 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000002-wal-2.temp at 1731558907265 (+30 ms)3 split writer threads finished at 1731558907266 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731558907277 (+11 ms)Processed 1 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731558907308 (+31 ms) 2024-11-14T04:35:07,309 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:07,311 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:07,323 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal.1731558907312, exclude list is [], retry=0 2024-11-14T04:35:07,328 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:07,328 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:07,329 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:07,354 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal.1731558907312 2024-11-14T04:35:07,354 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-14T04:35:07,354 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 048f68c571159453a72905c3b4cf6207, NAME => 'testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:07,355 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:07,355 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:07,355 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:07,357 INFO [StoreOpener-048f68c571159453a72905c3b4cf6207-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:07,359 INFO [StoreOpener-048f68c571159453a72905c3b4cf6207-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 048f68c571159453a72905c3b4cf6207 columnFamilyName a 2024-11-14T04:35:07,359 DEBUG [StoreOpener-048f68c571159453a72905c3b4cf6207-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:07,360 INFO [StoreOpener-048f68c571159453a72905c3b4cf6207-1 {}] regionserver.HStore(327): Store=048f68c571159453a72905c3b4cf6207/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:07,360 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:07,361 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:07,364 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:07,365 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000002 2024-11-14T04:35:07,392 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:07,397 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000002 2024-11-14T04:35:07,400 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 048f68c571159453a72905c3b4cf6207 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-14T04:35:07,448 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/.tmp/a/405b06567d64473c802aedd58d745fda is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731558906992/Put/seqid=0 2024-11-14T04:35:07,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741844_1020 (size=5170) 2024-11-14T04:35:07,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741844_1020 (size=5170) 2024-11-14T04:35:07,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741844_1020 (size=5170) 2024-11-14T04:35:07,461 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/.tmp/a/405b06567d64473c802aedd58d745fda 2024-11-14T04:35:07,501 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/.tmp/a/405b06567d64473c802aedd58d745fda as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/a/405b06567d64473c802aedd58d745fda 2024-11-14T04:35:07,510 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/a/405b06567d64473c802aedd58d745fda, entries=2, sequenceid=2, filesize=5.0 K 2024-11-14T04:35:07,516 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 048f68c571159453a72905c3b4cf6207 in 114ms, sequenceid=2, compaction requested=false; wal=null 2024-11-14T04:35:07,518 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/0000000000000000002 2024-11-14T04:35:07,519 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:07,519 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:07,522 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 048f68c571159453a72905c3b4cf6207 2024-11-14T04:35:07,525 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/048f68c571159453a72905c3b4cf6207/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-14T04:35:07,526 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 048f68c571159453a72905c3b4cf6207; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68885668, jitterRate=0.02647644281387329}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:35:07,527 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 048f68c571159453a72905c3b4cf6207: Writing region info on filesystem at 1731558907355Initializing all the Stores at 1731558907357 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558907357Obtaining lock to block concurrent updates at 1731558907400 (+43 ms)Preparing flush snapshotting stores in 048f68c571159453a72905c3b4cf6207 at 1731558907400Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731558907403 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731558906889.048f68c571159453a72905c3b4cf6207. at 1731558907403Flushing 048f68c571159453a72905c3b4cf6207/a: creating writer at 1731558907404 (+1 ms)Flushing 048f68c571159453a72905c3b4cf6207/a: appending metadata at 1731558907437 (+33 ms)Flushing 048f68c571159453a72905c3b4cf6207/a: closing flushed file at 1731558907439 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@899f4d5: reopening flushed file at 1731558907499 (+60 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 048f68c571159453a72905c3b4cf6207 in 114ms, sequenceid=2, compaction requested=false; wal=null at 1731558907516 (+17 ms)Cleaning up temporary data from old regions at 1731558907519 (+3 ms)Region opened successfully at 1731558907527 (+8 ms) 2024-11-14T04:35:07,555 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=376 (was 365) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:60980 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:32810 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:38630 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:41731/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:42028 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:38822 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:42144 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=464 (was 464), ProcessCount=11 (was 11), AvailableMemoryMB=7101 (was 7123) 2024-11-14T04:35:07,568 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=464, ProcessCount=11, AvailableMemoryMB=7101 2024-11-14T04:35:07,585 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:07,588 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:07,589 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:07,593 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-22155971, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/hregion-22155971, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:07,605 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-22155971/hregion-22155971.1731558907593, exclude list is [], retry=0 2024-11-14T04:35:07,609 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:07,609 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:07,610 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:07,636 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-22155971/hregion-22155971.1731558907593 2024-11-14T04:35:07,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-14T04:35:07,637 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 03d05f11e902b43931ca1cbe279deee1, NAME => 'testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/hbase 2024-11-14T04:35:07,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741846_1022 (size=64) 2024-11-14T04:35:07,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741846_1022 (size=64) 2024-11-14T04:35:07,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741846_1022 (size=64) 2024-11-14T04:35:07,652 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:07,654 INFO [StoreOpener-03d05f11e902b43931ca1cbe279deee1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:07,656 INFO [StoreOpener-03d05f11e902b43931ca1cbe279deee1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03d05f11e902b43931ca1cbe279deee1 columnFamilyName a 2024-11-14T04:35:07,656 DEBUG [StoreOpener-03d05f11e902b43931ca1cbe279deee1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:07,657 INFO [StoreOpener-03d05f11e902b43931ca1cbe279deee1-1 {}] regionserver.HStore(327): Store=03d05f11e902b43931ca1cbe279deee1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:07,657 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:07,658 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:07,658 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:07,659 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:07,659 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:07,661 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:07,665 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:07,666 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 03d05f11e902b43931ca1cbe279deee1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69039580, jitterRate=0.02876991033554077}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:35:07,666 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 03d05f11e902b43931ca1cbe279deee1: Writing region info on filesystem at 1731558907652Initializing all the Stores at 1731558907653 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558907653Cleaning up temporary data from old regions at 1731558907659 (+6 ms)Region opened successfully at 1731558907666 (+7 ms) 2024-11-14T04:35:07,666 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 03d05f11e902b43931ca1cbe279deee1, disabling compactions & flushes 2024-11-14T04:35:07,666 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1. 2024-11-14T04:35:07,666 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1. 2024-11-14T04:35:07,666 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1. after waiting 0 ms 2024-11-14T04:35:07,667 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1. 2024-11-14T04:35:07,667 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1. 2024-11-14T04:35:07,667 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 03d05f11e902b43931ca1cbe279deee1: Waiting for close lock at 1731558907666Disabling compacts and flushes for region at 1731558907666Disabling writes for close at 1731558907666Writing region close event to WAL at 1731558907667 (+1 ms)Closed at 1731558907667 2024-11-14T04:35:07,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741845_1021 (size=175) 2024-11-14T04:35:07,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741845_1021 (size=175) 2024-11-14T04:35:07,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741845_1021 (size=175) 2024-11-14T04:35:07,673 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:07,674 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-22155971:(num 1731558907593) 2024-11-14T04:35:07,674 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T04:35:07,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741847_1023 (size=469) 2024-11-14T04:35:07,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741847_1023 (size=469) 2024-11-14T04:35:07,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741847_1023 (size=469) 2024-11-14T04:35:07,713 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T04:35:07,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741848_1024 (size=322) 2024-11-14T04:35:07,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741848_1024 (size=322) 2024-11-14T04:35:07,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741848_1024 (size=322) 2024-11-14T04:35:07,766 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-2, size=322 (322bytes) 2024-11-14T04:35:07,767 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-2 2024-11-14T04:35:07,768 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-2 after 1ms 2024-11-14T04:35:07,794 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:07,794 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-2 took 28ms 2024-11-14T04:35:07,797 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-2 so closing down 2024-11-14T04:35:07,797 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:07,799 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-14T04:35:07,825 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002-wal-2.temp 2024-11-14T04:35:07,826 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:07,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741849_1025 (size=322) 2024-11-14T04:35:07,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741849_1025 (size=322) 2024-11-14T04:35:07,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741849_1025 (size=322) 2024-11-14T04:35:07,835 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:07,837 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002 2024-11-14T04:35:07,837 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-14T04:35:07,837 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-2, journal: Splitting hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-2, size=322 (322bytes) at 1731558907766Finishing writing output for hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-2 so closing down at 1731558907797 (+31 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002-wal-2.temp at 1731558907825 (+28 ms)3 split writer threads finished at 1731558907826 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731558907835 (+9 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002 at 1731558907837 (+2 ms)Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731558907837 2024-11-14T04:35:07,854 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-1, size=469 (469bytes) 2024-11-14T04:35:07,854 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-1 2024-11-14T04:35:07,854 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-1 after 0ms 2024-11-14T04:35:07,882 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:07,882 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-1 took 29ms 2024-11-14T04:35:07,886 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-1 so closing down 2024-11-14T04:35:07,886 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:07,889 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-14T04:35:07,915 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000001-wal-1.temp 2024-11-14T04:35:07,915 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:07,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741850_1026 (size=469) 2024-11-14T04:35:07,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741850_1026 (size=469) 2024-11-14T04:35:07,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741850_1026 (size=469) 2024-11-14T04:35:07,923 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:07,950 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:07,953 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002, length=322 2024-11-14T04:35:07,955 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002 2024-11-14T04:35:07,955 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-14T04:35:07,955 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-1, journal: Splitting hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-1, size=469 (469bytes) at 1731558907854Finishing writing output for hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-1 so closing down at 1731558907886 (+32 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000001-wal-1.temp at 1731558907915 (+29 ms)3 split writer threads finished at 1731558907915Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731558907923 (+8 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002 at 1731558907955 (+32 ms)Processed 2 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731558907955 2024-11-14T04:35:07,955 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:07,957 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:07,969 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal.1731558907958, exclude list is [], retry=0 2024-11-14T04:35:07,973 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:07,973 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:07,973 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:08,000 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal.1731558907958 2024-11-14T04:35:08,000 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855)] 2024-11-14T04:35:08,001 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 03d05f11e902b43931ca1cbe279deee1, NAME => 'testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:08,001 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:08,001 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:08,001 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:08,003 INFO [StoreOpener-03d05f11e902b43931ca1cbe279deee1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:08,005 INFO [StoreOpener-03d05f11e902b43931ca1cbe279deee1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03d05f11e902b43931ca1cbe279deee1 columnFamilyName a 2024-11-14T04:35:08,005 DEBUG [StoreOpener-03d05f11e902b43931ca1cbe279deee1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:08,006 INFO [StoreOpener-03d05f11e902b43931ca1cbe279deee1-1 {}] regionserver.HStore(327): Store=03d05f11e902b43931ca1cbe279deee1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:08,006 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:08,007 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:08,009 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:08,010 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002 2024-11-14T04:35:08,037 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:08,039 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002 2024-11-14T04:35:08,039 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 03d05f11e902b43931ca1cbe279deee1 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-14T04:35:08,058 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/.tmp/a/eb4762392b5449cf90db0a15d364b5b0 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731558907674/Put/seqid=0 2024-11-14T04:35:08,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741852_1028 (size=5170) 2024-11-14T04:35:08,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741852_1028 (size=5170) 2024-11-14T04:35:08,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741852_1028 (size=5170) 2024-11-14T04:35:08,066 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/.tmp/a/eb4762392b5449cf90db0a15d364b5b0 2024-11-14T04:35:08,076 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/.tmp/a/eb4762392b5449cf90db0a15d364b5b0 as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/a/eb4762392b5449cf90db0a15d364b5b0 2024-11-14T04:35:08,085 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/a/eb4762392b5449cf90db0a15d364b5b0, entries=2, sequenceid=2, filesize=5.0 K 2024-11-14T04:35:08,085 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 03d05f11e902b43931ca1cbe279deee1 in 46ms, sequenceid=2, compaction requested=false; wal=null 2024-11-14T04:35:08,086 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/0000000000000000002 2024-11-14T04:35:08,087 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:08,087 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:08,089 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 03d05f11e902b43931ca1cbe279deee1 2024-11-14T04:35:08,092 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/03d05f11e902b43931ca1cbe279deee1/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-14T04:35:08,093 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 03d05f11e902b43931ca1cbe279deee1; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61022754, jitterRate=-0.09069010615348816}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:35:08,094 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 03d05f11e902b43931ca1cbe279deee1: Writing region info on filesystem at 1731558908001Initializing all the Stores at 1731558908002 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558908002Obtaining lock to block concurrent updates at 1731558908039 (+37 ms)Preparing flush snapshotting stores in 03d05f11e902b43931ca1cbe279deee1 at 1731558908039Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731558908039Flushing stores of testReplayEditsWrittenIntoWAL,,1731558907586.03d05f11e902b43931ca1cbe279deee1. at 1731558908039Flushing 03d05f11e902b43931ca1cbe279deee1/a: creating writer at 1731558908039Flushing 03d05f11e902b43931ca1cbe279deee1/a: appending metadata at 1731558908056 (+17 ms)Flushing 03d05f11e902b43931ca1cbe279deee1/a: closing flushed file at 1731558908056Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@590d597c: reopening flushed file at 1731558908074 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 03d05f11e902b43931ca1cbe279deee1 in 46ms, sequenceid=2, compaction requested=false; wal=null at 1731558908085 (+11 ms)Cleaning up temporary data from old regions at 1731558908087 (+2 ms)Region opened successfully at 1731558908093 (+6 ms) 2024-11-14T04:35:08,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:42028 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:38896 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:60980 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:42186 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:38630 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:32866 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=464 (was 464), ProcessCount=11 (was 11), AvailableMemoryMB=7085 (was 7101) 2024-11-14T04:35:08,124 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=464, ProcessCount=11, AvailableMemoryMB=7085 2024-11-14T04:35:08,142 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:08,144 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:08,144 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:08,148 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-55774911, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/hregion-55774911, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:08,159 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-55774911/hregion-55774911.1731558908148, exclude list is [], retry=0 2024-11-14T04:35:08,163 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:08,163 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:08,163 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:08,190 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-55774911/hregion-55774911.1731558908148 2024-11-14T04:35:08,190 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-14T04:35:08,190 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 39db97f17e94abd9168ae904a9a3127c, NAME => 'testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/hbase 2024-11-14T04:35:08,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741854_1030 (size=64) 2024-11-14T04:35:08,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741854_1030 (size=64) 2024-11-14T04:35:08,202 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:08,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741854_1030 (size=64) 2024-11-14T04:35:08,203 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:08,205 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39db97f17e94abd9168ae904a9a3127c columnFamilyName a 2024-11-14T04:35:08,205 DEBUG [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:08,206 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(327): Store=39db97f17e94abd9168ae904a9a3127c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:08,206 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:08,207 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39db97f17e94abd9168ae904a9a3127c columnFamilyName b 2024-11-14T04:35:08,207 DEBUG [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:08,208 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(327): Store=39db97f17e94abd9168ae904a9a3127c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:08,208 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:08,210 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39db97f17e94abd9168ae904a9a3127c columnFamilyName c 2024-11-14T04:35:08,210 DEBUG [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:08,210 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(327): Store=39db97f17e94abd9168ae904a9a3127c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:08,211 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:08,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:08,212 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:08,213 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:08,213 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:08,213 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:08,215 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:08,217 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:08,218 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 39db97f17e94abd9168ae904a9a3127c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63442336, jitterRate=-0.05463552474975586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:08,218 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 39db97f17e94abd9168ae904a9a3127c: Writing region info on filesystem at 1731558908202Initializing all the Stores at 1731558908203 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558908203Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558908203Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558908203Cleaning up temporary data from old regions at 1731558908213 (+10 ms)Region opened successfully at 1731558908218 (+5 ms) 2024-11-14T04:35:08,218 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 39db97f17e94abd9168ae904a9a3127c, disabling compactions & flushes 2024-11-14T04:35:08,218 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. 2024-11-14T04:35:08,218 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. 2024-11-14T04:35:08,218 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. after waiting 0 ms 2024-11-14T04:35:08,218 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. 2024-11-14T04:35:08,219 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. 2024-11-14T04:35:08,219 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 39db97f17e94abd9168ae904a9a3127c: Waiting for close lock at 1731558908218Disabling compacts and flushes for region at 1731558908218Disabling writes for close at 1731558908218Writing region close event to WAL at 1731558908219 (+1 ms)Closed at 1731558908219 2024-11-14T04:35:08,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741853_1029 (size=175) 2024-11-14T04:35:08,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741853_1029 (size=175) 2024-11-14T04:35:08,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741853_1029 (size=175) 2024-11-14T04:35:08,225 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:08,226 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-55774911:(num 1731558908148) 2024-11-14T04:35:08,226 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:08,228 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:08,241 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229, exclude list is [], retry=0 2024-11-14T04:35:08,244 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:08,245 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:08,245 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:08,271 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 2024-11-14T04:35:08,272 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:08,510 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229, size=0 (0bytes) 2024-11-14T04:35:08,510 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 might be still open, length is 0 2024-11-14T04:35:08,510 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 2024-11-14T04:35:08,511 WARN [IPC Server handler 3 on default port 41731 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-14T04:35:08,512 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 after 2ms 2024-11-14T04:35:11,517 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:32906 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:32906 dst: /127.0.0.1:33533 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33533 remote=/127.0.0.1:32906]. Total timeout mills is 60000, 56953 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:11,518 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:42220 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42220 dst: /127.0.0.1:36191 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:11,518 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:38930 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37377:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38930 dst: /127.0.0.1:37377 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:11,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741855_1032 (size=497775) 2024-11-14T04:35:11,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741855_1032 (size=497775) 2024-11-14T04:35:11,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741855_1032 (size=497775) 2024-11-14T04:35:11,948 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T04:35:12,003 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T04:35:12,514 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 after 4004ms 2024-11-14T04:35:12,553 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:12,554 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 took 4044ms 2024-11-14T04:35:12,559 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731558908229.temp 2024-11-14T04:35:12,587 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000000001-wal.1731558908229.temp 2024-11-14T04:35:12,666 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229; continuing. 2024-11-14T04:35:12,666 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 so closing down 2024-11-14T04:35:12,666 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:12,666 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:12,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741856_1033 (size=497783) 2024-11-14T04:35:12,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741856_1033 (size=497783) 2024-11-14T04:35:12,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741856_1033 (size=497783) 2024-11-14T04:35:12,671 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000000001-wal.1731558908229.temp (wrote 3002 edits, skipped 0 edits in 44 ms) 2024-11-14T04:35:12,673 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000000001-wal.1731558908229.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000003002 2024-11-14T04:35:12,673 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 118 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T04:35:12,673 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229, journal: Splitting hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229, size=0 (0bytes) at 1731558908510Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000000001-wal.1731558908229.temp at 1731558912587 (+4077 ms)Split 1024 edits, skipped 0 edits. at 1731558912621 (+34 ms)Split 2048 edits, skipped 0 edits. at 1731558912646 (+25 ms)Finishing writing output for hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 so closing down at 1731558912666 (+20 ms)3 split writer threads finished at 1731558912666Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000000001-wal.1731558908229.temp (wrote 3002 edits, skipped 0 edits in 44 ms) at 1731558912671 (+5 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000000001-wal.1731558908229.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000003002 at 1731558912673 (+2 ms)Processed 3002 edits across 1 Regions in 118 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229, size=0, length=0, corrupted=false, cancelled=false at 1731558912673 2024-11-14T04:35:12,675 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 to hdfs://localhost:41731/hbase/oldWALs/wal.1731558908229 2024-11-14T04:35:12,677 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000003002 2024-11-14T04:35:12,677 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:12,679 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:12,696 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558912680, exclude list is [], retry=0 2024-11-14T04:35:12,699 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:12,700 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:12,700 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:12,734 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558912680 2024-11-14T04:35:12,734 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:12,734 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:12,736 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:12,737 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39db97f17e94abd9168ae904a9a3127c columnFamilyName a 2024-11-14T04:35:12,737 DEBUG [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:12,738 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(327): Store=39db97f17e94abd9168ae904a9a3127c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:12,738 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:12,740 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39db97f17e94abd9168ae904a9a3127c columnFamilyName b 2024-11-14T04:35:12,740 DEBUG [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:12,740 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(327): Store=39db97f17e94abd9168ae904a9a3127c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:12,740 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:12,742 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39db97f17e94abd9168ae904a9a3127c columnFamilyName c 2024-11-14T04:35:12,742 DEBUG [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:12,743 INFO [StoreOpener-39db97f17e94abd9168ae904a9a3127c-1 {}] regionserver.HStore(327): Store=39db97f17e94abd9168ae904a9a3127c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:12,743 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:12,744 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:12,746 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:12,747 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000003002 2024-11-14T04:35:12,778 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:12,822 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T04:35:13,162 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39db97f17e94abd9168ae904a9a3127c 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T04:35:13,200 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/a/2488d065f0a045c5bf505da3d2403113 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731558908279/Put/seqid=0 2024-11-14T04:35:13,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741858_1035 (size=50463) 2024-11-14T04:35:13,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741858_1035 (size=50463) 2024-11-14T04:35:13,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741858_1035 (size=50463) 2024-11-14T04:35:13,208 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/a/2488d065f0a045c5bf505da3d2403113 2024-11-14T04:35:13,216 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/a/2488d065f0a045c5bf505da3d2403113 as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/a/2488d065f0a045c5bf505da3d2403113 2024-11-14T04:35:13,223 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/a/2488d065f0a045c5bf505da3d2403113, entries=754, sequenceid=754, filesize=49.3 K 2024-11-14T04:35:13,223 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 39db97f17e94abd9168ae904a9a3127c in 62ms, sequenceid=754, compaction requested=false; wal=null 2024-11-14T04:35:13,237 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T04:35:13,237 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39db97f17e94abd9168ae904a9a3127c 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T04:35:13,247 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/a/79dc85dffc2040a48286599924bf61da is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731558908317/Put/seqid=0 2024-11-14T04:35:13,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741859_1036 (size=20072) 2024-11-14T04:35:13,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741859_1036 (size=20072) 2024-11-14T04:35:13,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741859_1036 (size=20072) 2024-11-14T04:35:13,257 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/a/79dc85dffc2040a48286599924bf61da 2024-11-14T04:35:13,281 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/b/8c741e38faa24393897db13e71aa58cf is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731558908350/Put/seqid=0 2024-11-14T04:35:13,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741860_1037 (size=35835) 2024-11-14T04:35:13,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741860_1037 (size=35835) 2024-11-14T04:35:13,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741860_1037 (size=35835) 2024-11-14T04:35:13,289 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/b/8c741e38faa24393897db13e71aa58cf 2024-11-14T04:35:13,296 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/a/79dc85dffc2040a48286599924bf61da as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/a/79dc85dffc2040a48286599924bf61da 2024-11-14T04:35:13,303 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/a/79dc85dffc2040a48286599924bf61da, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-14T04:35:13,305 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/b/8c741e38faa24393897db13e71aa58cf as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/b/8c741e38faa24393897db13e71aa58cf 2024-11-14T04:35:13,313 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/b/8c741e38faa24393897db13e71aa58cf, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-14T04:35:13,313 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 39db97f17e94abd9168ae904a9a3127c in 76ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-14T04:35:13,326 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T04:35:13,327 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39db97f17e94abd9168ae904a9a3127c 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T04:35:13,334 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/b/30a66faf2266438d9610cb4f18ad7c2c is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731558908378/Put/seqid=0 2024-11-14T04:35:13,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741861_1038 (size=35082) 2024-11-14T04:35:13,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741861_1038 (size=35082) 2024-11-14T04:35:13,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741861_1038 (size=35082) 2024-11-14T04:35:13,344 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/b/30a66faf2266438d9610cb4f18ad7c2c 2024-11-14T04:35:13,376 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/c/1fd4d508748a4578af7d2433426897c4 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731558908416/Put/seqid=0 2024-11-14T04:35:13,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741862_1039 (size=20825) 2024-11-14T04:35:13,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741862_1039 (size=20825) 2024-11-14T04:35:13,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741862_1039 (size=20825) 2024-11-14T04:35:13,385 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/c/1fd4d508748a4578af7d2433426897c4 2024-11-14T04:35:13,392 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/b/30a66faf2266438d9610cb4f18ad7c2c as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/b/30a66faf2266438d9610cb4f18ad7c2c 2024-11-14T04:35:13,399 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/b/30a66faf2266438d9610cb4f18ad7c2c, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-14T04:35:13,400 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/c/1fd4d508748a4578af7d2433426897c4 as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/c/1fd4d508748a4578af7d2433426897c4 2024-11-14T04:35:13,407 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/c/1fd4d508748a4578af7d2433426897c4, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-14T04:35:13,407 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 39db97f17e94abd9168ae904a9a3127c in 81ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-14T04:35:13,423 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731558908468/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. 2024-11-14T04:35:13,427 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000003002 2024-11-14T04:35:13,427 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T04:35:13,428 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39db97f17e94abd9168ae904a9a3127c 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-14T04:35:13,437 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/c/15b3e6fd882f46cf8b6a50cb3a765c55 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731558908424/Put/seqid=0 2024-11-14T04:35:13,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741863_1040 (size=50301) 2024-11-14T04:35:13,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741863_1040 (size=50301) 2024-11-14T04:35:13,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741863_1040 (size=50301) 2024-11-14T04:35:13,449 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/c/15b3e6fd882f46cf8b6a50cb3a765c55 2024-11-14T04:35:13,455 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 15b3e6fd882f46cf8b6a50cb3a765c55 2024-11-14T04:35:13,456 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/.tmp/c/15b3e6fd882f46cf8b6a50cb3a765c55 as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/c/15b3e6fd882f46cf8b6a50cb3a765c55 2024-11-14T04:35:13,462 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 15b3e6fd882f46cf8b6a50cb3a765c55 2024-11-14T04:35:13,463 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/c/15b3e6fd882f46cf8b6a50cb3a765c55, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-14T04:35:13,463 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 39db97f17e94abd9168ae904a9a3127c in 35ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-14T04:35:13,464 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/0000000000000003002 2024-11-14T04:35:13,465 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:13,465 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:13,466 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T04:35:13,468 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 39db97f17e94abd9168ae904a9a3127c 2024-11-14T04:35:13,472 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenIntoWAL/39db97f17e94abd9168ae904a9a3127c/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-14T04:35:13,473 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 39db97f17e94abd9168ae904a9a3127c; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66204343, jitterRate=-0.013478413224220276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T04:35:13,474 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 39db97f17e94abd9168ae904a9a3127c: Writing region info on filesystem at 1731558912735Initializing all the Stores at 1731558912736 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558912736Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558912736Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558912736Cleaning up temporary data from old regions at 1731558913465 (+729 ms)Region opened successfully at 1731558913473 (+8 ms) 2024-11-14T04:35:13,557 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 39db97f17e94abd9168ae904a9a3127c, disabling compactions & flushes 2024-11-14T04:35:13,557 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. 2024-11-14T04:35:13,557 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. 2024-11-14T04:35:13,557 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. after waiting 0 ms 2024-11-14T04:35:13,557 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. 2024-11-14T04:35:13,559 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731558908143.39db97f17e94abd9168ae904a9a3127c. 2024-11-14T04:35:13,559 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 39db97f17e94abd9168ae904a9a3127c: Waiting for close lock at 1731558913557Disabling compacts and flushes for region at 1731558913557Disabling writes for close at 1731558913557Writing region close event to WAL at 1731558913559 (+2 ms)Closed at 1731558913559 2024-11-14T04:35:13,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741857_1034 (size=175) 2024-11-14T04:35:13,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741857_1034 (size=175) 2024-11-14T04:35:13,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741857_1034 (size=175) 2024-11-14T04:35:13,566 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:13,566 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731558912680) 2024-11-14T04:35:13,580 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=400 (was 386) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1423948243_22 at /127.0.0.1:41368 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:41731 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1423948243_22 at /127.0.0.1:49748 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35041 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:41257 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41257 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1423948243_22 at /127.0.0.1:41402 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35041 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1423948243_22 at /127.0.0.1:47766 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:41731 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 464), ProcessCount=11 (was 11), AvailableMemoryMB=6990 (was 7085) 2024-11-14T04:35:13,593 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=400, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=6989 2024-11-14T04:35:13,616 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:13,620 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:13,621 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:13,625 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-29698492, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/hregion-29698492, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:13,639 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-29698492/hregion-29698492.1731558913625, exclude list is [], retry=0 2024-11-14T04:35:13,643 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:13,643 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:13,644 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:13,677 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-29698492/hregion-29698492.1731558913625 2024-11-14T04:35:13,678 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:13,678 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 00fe04730fa9792cb9db0a6e72f95ae3, NAME => 'test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/hbase 2024-11-14T04:35:13,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741865_1042 (size=43) 2024-11-14T04:35:13,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741865_1042 (size=43) 2024-11-14T04:35:13,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741865_1042 (size=43) 2024-11-14T04:35:13,689 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:13,690 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:13,691 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00fe04730fa9792cb9db0a6e72f95ae3 columnFamilyName a 2024-11-14T04:35:13,692 DEBUG [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:13,692 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(327): Store=00fe04730fa9792cb9db0a6e72f95ae3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:13,692 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:13,694 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00fe04730fa9792cb9db0a6e72f95ae3 columnFamilyName b 2024-11-14T04:35:13,695 DEBUG [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:13,695 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(327): Store=00fe04730fa9792cb9db0a6e72f95ae3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:13,696 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:13,697 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00fe04730fa9792cb9db0a6e72f95ae3 columnFamilyName c 2024-11-14T04:35:13,697 DEBUG [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:13,698 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(327): Store=00fe04730fa9792cb9db0a6e72f95ae3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:13,698 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:13,699 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:13,699 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:13,701 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:13,701 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:13,701 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:13,702 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:13,705 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:13,706 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 00fe04730fa9792cb9db0a6e72f95ae3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60804208, jitterRate=-0.09394669532775879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:13,708 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 00fe04730fa9792cb9db0a6e72f95ae3: Writing region info on filesystem at 1731558913689Initializing all the Stores at 1731558913690 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558913690Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558913690Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558913690Cleaning up temporary data from old regions at 1731558913701 (+11 ms)Region opened successfully at 1731558913707 (+6 ms) 2024-11-14T04:35:13,708 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 00fe04730fa9792cb9db0a6e72f95ae3, disabling compactions & flushes 2024-11-14T04:35:13,708 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. 2024-11-14T04:35:13,708 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. 2024-11-14T04:35:13,708 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. after waiting 0 ms 2024-11-14T04:35:13,708 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. 2024-11-14T04:35:13,709 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. 2024-11-14T04:35:13,709 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 00fe04730fa9792cb9db0a6e72f95ae3: Waiting for close lock at 1731558913708Disabling compacts and flushes for region at 1731558913708Disabling writes for close at 1731558913708Writing region close event to WAL at 1731558913709 (+1 ms)Closed at 1731558913709 2024-11-14T04:35:13,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741864_1041 (size=175) 2024-11-14T04:35:13,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741864_1041 (size=175) 2024-11-14T04:35:13,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741864_1041 (size=175) 2024-11-14T04:35:13,718 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:13,718 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-29698492:(num 1731558913625) 2024-11-14T04:35:13,718 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:13,720 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:13,733 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721, exclude list is [], retry=0 2024-11-14T04:35:13,737 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:13,737 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:13,737 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:13,767 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721 2024-11-14T04:35:13,768 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:13,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T04:35:13,890 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T04:35:13,892 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T04:35:13,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T04:35:13,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T04:35:13,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T04:35:13,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-14T04:35:13,894 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-14T04:35:13,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741866_1043 (size=371388) 2024-11-14T04:35:13,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741866_1043 (size=371388) 2024-11-14T04:35:13,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741866_1043 (size=371388) 2024-11-14T04:35:13,985 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721, size=362.7 K (371388bytes) 2024-11-14T04:35:13,985 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721 2024-11-14T04:35:13,986 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721 after 0ms 2024-11-14T04:35:14,024 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:14,026 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721 took 40ms 2024-11-14T04:35:14,029 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731558913721.temp 2024-11-14T04:35:14,067 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721 so closing down 2024-11-14T04:35:14,067 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:14,068 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000000001-wal.1731558913721.temp 2024-11-14T04:35:14,082 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:14,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741867_1044 (size=371388) 2024-11-14T04:35:14,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741867_1044 (size=371388) 2024-11-14T04:35:14,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741867_1044 (size=371388) 2024-11-14T04:35:14,086 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000000001-wal.1731558913721.temp (wrote 3000 edits, skipped 0 edits in 13 ms) 2024-11-14T04:35:14,088 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000000001-wal.1731558913721.temp to hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003000 2024-11-14T04:35:14,088 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-14T04:35:14,088 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721, journal: Splitting hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721, size=362.7 K (371388bytes) at 1731558913985Split 1024 edits, skipped 0 edits. at 1731558914042 (+57 ms)Split 2048 edits, skipped 0 edits. at 1731558914055 (+13 ms)Finishing writing output for hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721 so closing down at 1731558914067 (+12 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000000001-wal.1731558913721.temp at 1731558914068 (+1 ms)3 split writer threads finished at 1731558914082 (+14 ms)Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000000001-wal.1731558913721.temp (wrote 3000 edits, skipped 0 edits in 13 ms) at 1731558914086 (+4 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000000001-wal.1731558913721.temp to hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003000 at 1731558914088 (+2 ms)Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1731558914088 2024-11-14T04:35:14,090 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558913721 to hdfs://localhost:41731/hbase/oldWALs/wal.1731558913721 2024-11-14T04:35:14,091 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003000 2024-11-14T04:35:14,091 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:14,094 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:14,107 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094, exclude list is [], retry=0 2024-11-14T04:35:14,111 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:14,111 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:14,112 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:14,147 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094 2024-11-14T04:35:14,148 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:14,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741868_1045 (size=371515) 2024-11-14T04:35:14,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741868_1045 (size=371515) 2024-11-14T04:35:14,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741868_1045 (size=371515) 2024-11-14T04:35:14,332 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094, size=362.8 K (371515bytes) 2024-11-14T04:35:14,332 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094 2024-11-14T04:35:14,333 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094 after 1ms 2024-11-14T04:35:14,365 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:14,367 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094 took 35ms 2024-11-14T04:35:14,372 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731558914094.temp 2024-11-14T04:35:14,427 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094 so closing down 2024-11-14T04:35:14,427 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:14,446 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003001-wal.1731558914094.temp 2024-11-14T04:35:14,464 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:14,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741869_1046 (size=371515) 2024-11-14T04:35:14,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741869_1046 (size=371515) 2024-11-14T04:35:14,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741869_1046 (size=371515) 2024-11-14T04:35:14,481 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003001-wal.1731558914094.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-11-14T04:35:14,483 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003001-wal.1731558914094.temp to hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000006000 2024-11-14T04:35:14,484 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 116 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-14T04:35:14,484 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094, journal: Splitting hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094, size=362.8 K (371515bytes) at 1731558914332Split 1024 edits, skipped 0 edits. at 1731558914380 (+48 ms)Split 2048 edits, skipped 0 edits. at 1731558914420 (+40 ms)Finishing writing output for hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094 so closing down at 1731558914427 (+7 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003001-wal.1731558914094.temp at 1731558914446 (+19 ms)3 split writer threads finished at 1731558914464 (+18 ms)Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003001-wal.1731558914094.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1731558914481 (+17 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003001-wal.1731558914094.temp to hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000006000 at 1731558914483 (+2 ms)Processed 3000 edits across 1 Regions in 116 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1731558914484 (+1 ms) 2024-11-14T04:35:14,486 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914094 to hdfs://localhost:41731/hbase/oldWALs/wal.1731558914094 2024-11-14T04:35:14,488 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000006000 2024-11-14T04:35:14,488 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:14,490 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/test2727-manual,16010,1731558913615, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:14,511 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914491, exclude list is [], retry=0 2024-11-14T04:35:14,515 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:14,515 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:14,516 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:14,546 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914491 2024-11-14T04:35:14,547 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:46855:46855)] 2024-11-14T04:35:14,547 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 00fe04730fa9792cb9db0a6e72f95ae3, NAME => 'test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:14,547 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:14,548 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,548 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,549 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,551 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00fe04730fa9792cb9db0a6e72f95ae3 columnFamilyName a 2024-11-14T04:35:14,551 DEBUG [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:14,552 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(327): Store=00fe04730fa9792cb9db0a6e72f95ae3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:14,552 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,554 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00fe04730fa9792cb9db0a6e72f95ae3 columnFamilyName b 2024-11-14T04:35:14,554 DEBUG [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:14,555 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(327): Store=00fe04730fa9792cb9db0a6e72f95ae3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:14,555 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,556 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00fe04730fa9792cb9db0a6e72f95ae3 columnFamilyName c 2024-11-14T04:35:14,556 DEBUG [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:14,557 INFO [StoreOpener-00fe04730fa9792cb9db0a6e72f95ae3-1 {}] regionserver.HStore(327): Store=00fe04730fa9792cb9db0a6e72f95ae3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:14,557 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,558 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,561 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,562 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003000 2024-11-14T04:35:14,592 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:14,619 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003000 2024-11-14T04:35:14,620 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000006000 2024-11-14T04:35:14,646 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:14,672 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000006000 2024-11-14T04:35:14,672 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 00fe04730fa9792cb9db0a6e72f95ae3 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-14T04:35:14,701 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/.tmp/a/5818fe0a5d3744ecb3fa4f3ae31156e8 is 41, key is test2727/a:100/1731558914153/Put/seqid=0 2024-11-14T04:35:14,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741871_1048 (size=84227) 2024-11-14T04:35:14,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741871_1048 (size=84227) 2024-11-14T04:35:14,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741871_1048 (size=84227) 2024-11-14T04:35:14,707 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/.tmp/a/5818fe0a5d3744ecb3fa4f3ae31156e8 2024-11-14T04:35:14,731 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/.tmp/b/259a198d9169450fa04902f2869fb4ca is 41, key is test2727/b:100/1731558914218/Put/seqid=0 2024-11-14T04:35:14,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741872_1049 (size=84609) 2024-11-14T04:35:14,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741872_1049 (size=84609) 2024-11-14T04:35:14,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741872_1049 (size=84609) 2024-11-14T04:35:14,737 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/.tmp/b/259a198d9169450fa04902f2869fb4ca 2024-11-14T04:35:14,761 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/.tmp/c/af45b3247b2a48468a9926acc4772fa3 is 41, key is test2727/c:100/1731558914271/Put/seqid=0 2024-11-14T04:35:14,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741873_1050 (size=84609) 2024-11-14T04:35:14,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741873_1050 (size=84609) 2024-11-14T04:35:14,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741873_1050 (size=84609) 2024-11-14T04:35:14,769 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/.tmp/c/af45b3247b2a48468a9926acc4772fa3 2024-11-14T04:35:14,777 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/.tmp/a/5818fe0a5d3744ecb3fa4f3ae31156e8 as hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/a/5818fe0a5d3744ecb3fa4f3ae31156e8 2024-11-14T04:35:14,784 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/a/5818fe0a5d3744ecb3fa4f3ae31156e8, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-14T04:35:14,785 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/.tmp/b/259a198d9169450fa04902f2869fb4ca as hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/b/259a198d9169450fa04902f2869fb4ca 2024-11-14T04:35:14,791 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/b/259a198d9169450fa04902f2869fb4ca, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-14T04:35:14,793 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/.tmp/c/af45b3247b2a48468a9926acc4772fa3 as hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/c/af45b3247b2a48468a9926acc4772fa3 2024-11-14T04:35:14,799 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/c/af45b3247b2a48468a9926acc4772fa3, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-14T04:35:14,800 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 00fe04730fa9792cb9db0a6e72f95ae3 in 127ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-14T04:35:14,800 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000003000 2024-11-14T04:35:14,801 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/0000000000000006000 2024-11-14T04:35:14,802 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,802 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,803 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:14,805 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 00fe04730fa9792cb9db0a6e72f95ae3 2024-11-14T04:35:14,807 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/test2727/00fe04730fa9792cb9db0a6e72f95ae3/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-14T04:35:14,808 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 00fe04730fa9792cb9db0a6e72f95ae3; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72894516, jitterRate=0.08621293306350708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:14,809 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 00fe04730fa9792cb9db0a6e72f95ae3: Writing region info on filesystem at 1731558914548Initializing all the Stores at 1731558914549 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558914549Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558914549Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558914549Obtaining lock to block concurrent updates at 1731558914672 (+123 ms)Preparing flush snapshotting stores in 00fe04730fa9792cb9db0a6e72f95ae3 at 1731558914672Finished memstore snapshotting test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731558914672Flushing stores of test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. at 1731558914672Flushing 00fe04730fa9792cb9db0a6e72f95ae3/a: creating writer at 1731558914672Flushing 00fe04730fa9792cb9db0a6e72f95ae3/a: appending metadata at 1731558914700 (+28 ms)Flushing 00fe04730fa9792cb9db0a6e72f95ae3/a: closing flushed file at 1731558914700Flushing 00fe04730fa9792cb9db0a6e72f95ae3/b: creating writer at 1731558914713 (+13 ms)Flushing 00fe04730fa9792cb9db0a6e72f95ae3/b: appending metadata at 1731558914729 (+16 ms)Flushing 00fe04730fa9792cb9db0a6e72f95ae3/b: closing flushed file at 1731558914729Flushing 00fe04730fa9792cb9db0a6e72f95ae3/c: creating writer at 1731558914743 (+14 ms)Flushing 00fe04730fa9792cb9db0a6e72f95ae3/c: appending metadata at 1731558914761 (+18 ms)Flushing 00fe04730fa9792cb9db0a6e72f95ae3/c: closing flushed file at 1731558914761Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c234ee6: reopening flushed file at 1731558914776 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@530f6a0f: reopening flushed file at 1731558914784 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a3cbc89: reopening flushed file at 1731558914792 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 00fe04730fa9792cb9db0a6e72f95ae3 in 127ms, sequenceid=6000, compaction requested=false; wal=null at 1731558914800 (+8 ms)Cleaning up temporary data from old regions at 1731558914802 (+2 ms)Region opened successfully at 1731558914809 (+7 ms) 2024-11-14T04:35:14,810 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-14T04:35:14,810 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 00fe04730fa9792cb9db0a6e72f95ae3, disabling compactions & flushes 2024-11-14T04:35:14,810 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. 2024-11-14T04:35:14,810 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. 2024-11-14T04:35:14,810 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. after waiting 0 ms 2024-11-14T04:35:14,810 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. 2024-11-14T04:35:14,812 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731558913617.00fe04730fa9792cb9db0a6e72f95ae3. 2024-11-14T04:35:14,812 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 00fe04730fa9792cb9db0a6e72f95ae3: Waiting for close lock at 1731558914810Disabling compacts and flushes for region at 1731558914810Disabling writes for close at 1731558914810Writing region close event to WAL at 1731558914811 (+1 ms)Closed at 1731558914811 2024-11-14T04:35:14,816 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1731558913615/wal.1731558914491 not finished, retry = 0 2024-11-14T04:35:14,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741870_1047 (size=175) 2024-11-14T04:35:14,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741870_1047 (size=175) 2024-11-14T04:35:14,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741870_1047 (size=175) 2024-11-14T04:35:14,919 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:14,919 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731558914491) 2024-11-14T04:35:14,932 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=404 (was 400) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:47882 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:49748 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:41472 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=6759 (was 6989) 2024-11-14T04:35:14,945 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=6759 2024-11-14T04:35:14,962 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:14,969 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:14,969 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731558914969 2024-11-14T04:35:15,000 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969 2024-11-14T04:35:15,003 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:15,005 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ed1233e8fa961eac4e8ca208040004e4, NAME => 'testSequentialEditLogSeqNum,,1731558914963.ed1233e8fa961eac4e8ca208040004e4.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:15,005 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731558914963.ed1233e8fa961eac4e8ca208040004e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:15,005 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ed1233e8fa961eac4e8ca208040004e4 2024-11-14T04:35:15,005 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ed1233e8fa961eac4e8ca208040004e4 2024-11-14T04:35:15,006 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4 doesn't exist for region: ed1233e8fa961eac4e8ca208040004e4 on table testSequentialEditLogSeqNum 2024-11-14T04:35:15,007 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: ed1233e8fa961eac4e8ca208040004e4 on table testSequentialEditLogSeqNum 2024-11-14T04:35:15,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741875_1052 (size=62) 2024-11-14T04:35:15,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741875_1052 (size=62) 2024-11-14T04:35:15,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741875_1052 (size=62) 2024-11-14T04:35:15,018 INFO [StoreOpener-ed1233e8fa961eac4e8ca208040004e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ed1233e8fa961eac4e8ca208040004e4 2024-11-14T04:35:15,019 INFO [StoreOpener-ed1233e8fa961eac4e8ca208040004e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed1233e8fa961eac4e8ca208040004e4 columnFamilyName a 2024-11-14T04:35:15,020 DEBUG [StoreOpener-ed1233e8fa961eac4e8ca208040004e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:15,020 INFO [StoreOpener-ed1233e8fa961eac4e8ca208040004e4-1 {}] regionserver.HStore(327): Store=ed1233e8fa961eac4e8ca208040004e4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:15,020 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ed1233e8fa961eac4e8ca208040004e4 2024-11-14T04:35:15,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4 2024-11-14T04:35:15,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4 2024-11-14T04:35:15,022 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ed1233e8fa961eac4e8ca208040004e4 2024-11-14T04:35:15,022 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ed1233e8fa961eac4e8ca208040004e4 2024-11-14T04:35:15,024 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ed1233e8fa961eac4e8ca208040004e4 2024-11-14T04:35:15,026 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:15,027 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ed1233e8fa961eac4e8ca208040004e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61936113, jitterRate=-0.07707999646663666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:35:15,028 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ed1233e8fa961eac4e8ca208040004e4: Writing region info on filesystem at 1731558915005Initializing all the Stores at 1731558915018 (+13 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558915018Cleaning up temporary data from old regions at 1731558915022 (+4 ms)Region opened successfully at 1731558915028 (+6 ms) 2024-11-14T04:35:15,040 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ed1233e8fa961eac4e8ca208040004e4 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-14T04:35:15,063 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/.tmp/a/8bef927244994350ba623694c81a41ad is 81, key is testSequentialEditLogSeqNum/a:x0/1731558915028/Put/seqid=0 2024-11-14T04:35:15,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741876_1053 (size=5833) 2024-11-14T04:35:15,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741876_1053 (size=5833) 2024-11-14T04:35:15,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741876_1053 (size=5833) 2024-11-14T04:35:15,075 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/.tmp/a/8bef927244994350ba623694c81a41ad 2024-11-14T04:35:15,082 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/.tmp/a/8bef927244994350ba623694c81a41ad as hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/a/8bef927244994350ba623694c81a41ad 2024-11-14T04:35:15,088 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/a/8bef927244994350ba623694c81a41ad, entries=10, sequenceid=13, filesize=5.7 K 2024-11-14T04:35:15,089 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ed1233e8fa961eac4e8ca208040004e4 in 49ms, sequenceid=13, compaction requested=false 2024-11-14T04:35:15,090 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ed1233e8fa961eac4e8ca208040004e4: 2024-11-14T04:35:15,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T04:35:15,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T04:35:15,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T04:35:15,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T04:35:15,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T04:35:15,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741874_1051 (size=3256) 2024-11-14T04:35:15,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741874_1051 (size=3256) 2024-11-14T04:35:15,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741874_1051 (size=3256) 2024-11-14T04:35:15,115 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969, size=3.2 K (3256bytes) 2024-11-14T04:35:15,116 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969 2024-11-14T04:35:15,116 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969 after 0ms 2024-11-14T04:35:15,143 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:15,143 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969 took 28ms 2024-11-14T04:35:15,146 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969 so closing down 2024-11-14T04:35:15,146 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:15,147 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731558914969.temp 2024-11-14T04:35:15,173 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/recovered.edits/0000000000000000003-wal.1731558914969.temp 2024-11-14T04:35:15,174 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:15,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741877_1054 (size=2650) 2024-11-14T04:35:15,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741877_1054 (size=2650) 2024-11-14T04:35:15,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741877_1054 (size=2650) 2024-11-14T04:35:15,184 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/recovered.edits/0000000000000000003-wal.1731558914969.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:15,185 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/recovered.edits/0000000000000000003-wal.1731558914969.temp to hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/recovered.edits/0000000000000000020 2024-11-14T04:35:15,186 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-14T04:35:15,186 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969, journal: Splitting hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969, size=3.2 K (3256bytes) at 1731558915115Finishing writing output for hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969 so closing down at 1731558915146 (+31 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/recovered.edits/0000000000000000003-wal.1731558914969.temp at 1731558915173 (+27 ms)3 split writer threads finished at 1731558915174 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/recovered.edits/0000000000000000003-wal.1731558914969.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731558915184 (+10 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/recovered.edits/0000000000000000003-wal.1731558914969.temp to hdfs://localhost:41731/hbase/data/default/testSequentialEditLogSeqNum/ed1233e8fa961eac4e8ca208040004e4/recovered.edits/0000000000000000020 at 1731558915186 (+2 ms)Processed 17 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:41731/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731558914961/wal.1731558914969, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1731558915186 2024-11-14T04:35:15,201 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:49748 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:41472 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=6759 (was 6759) 2024-11-14T04:35:15,214 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=6758 2024-11-14T04:35:15,229 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:15,231 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:15,253 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:15,256 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80238880, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/hregion-80238880, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:15,268 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80238880/hregion-80238880.1731558915256, exclude list is [], retry=0 2024-11-14T04:35:15,272 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:15,272 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:15,272 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:15,299 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80238880/hregion-80238880.1731558915256 2024-11-14T04:35:15,300 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:46855:46855)] 2024-11-14T04:35:15,300 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7aa887e1dc72e692703c2af38a960d90, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/hbase 2024-11-14T04:35:15,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741879_1056 (size=70) 2024-11-14T04:35:15,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741879_1056 (size=70) 2024-11-14T04:35:15,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741879_1056 (size=70) 2024-11-14T04:35:15,311 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:15,312 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,314 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa887e1dc72e692703c2af38a960d90 columnFamilyName a 2024-11-14T04:35:15,314 DEBUG [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:15,314 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(327): Store=7aa887e1dc72e692703c2af38a960d90/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:15,315 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,316 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa887e1dc72e692703c2af38a960d90 columnFamilyName b 2024-11-14T04:35:15,316 DEBUG [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:15,317 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(327): Store=7aa887e1dc72e692703c2af38a960d90/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:15,317 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,319 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa887e1dc72e692703c2af38a960d90 columnFamilyName c 2024-11-14T04:35:15,319 DEBUG [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:15,320 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(327): Store=7aa887e1dc72e692703c2af38a960d90/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:15,320 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,321 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,321 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,322 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,322 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,323 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:15,324 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,327 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:15,327 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7aa887e1dc72e692703c2af38a960d90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73895457, jitterRate=0.10112811625003815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:15,328 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7aa887e1dc72e692703c2af38a960d90: Writing region info on filesystem at 1731558915311Initializing all the Stores at 1731558915312 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558915312Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558915312Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558915312Cleaning up temporary data from old regions at 1731558915322 (+10 ms)Region opened successfully at 1731558915328 (+6 ms) 2024-11-14T04:35:15,328 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7aa887e1dc72e692703c2af38a960d90, disabling compactions & flushes 2024-11-14T04:35:15,328 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. 2024-11-14T04:35:15,328 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. 2024-11-14T04:35:15,328 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. after waiting 0 ms 2024-11-14T04:35:15,328 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. 2024-11-14T04:35:15,329 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. 2024-11-14T04:35:15,329 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7aa887e1dc72e692703c2af38a960d90: Waiting for close lock at 1731558915328Disabling compacts and flushes for region at 1731558915328Disabling writes for close at 1731558915328Writing region close event to WAL at 1731558915329 (+1 ms)Closed at 1731558915329 2024-11-14T04:35:15,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741878_1055 (size=175) 2024-11-14T04:35:15,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741878_1055 (size=175) 2024-11-14T04:35:15,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741878_1055 (size=175) 2024-11-14T04:35:15,335 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:15,335 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80238880:(num 1731558915256) 2024-11-14T04:35:15,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:15,337 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:15,350 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337, exclude list is [], retry=0 2024-11-14T04:35:15,353 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:15,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:15,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:15,380 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 2024-11-14T04:35:15,381 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:15,381 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7aa887e1dc72e692703c2af38a960d90, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:15,381 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:15,381 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,381 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,383 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,384 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa887e1dc72e692703c2af38a960d90 columnFamilyName a 2024-11-14T04:35:15,384 DEBUG [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:15,385 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(327): Store=7aa887e1dc72e692703c2af38a960d90/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:15,385 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,385 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa887e1dc72e692703c2af38a960d90 columnFamilyName b 2024-11-14T04:35:15,385 DEBUG [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:15,386 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(327): Store=7aa887e1dc72e692703c2af38a960d90/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:15,386 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,387 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa887e1dc72e692703c2af38a960d90 columnFamilyName c 2024-11-14T04:35:15,387 DEBUG [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:15,387 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(327): Store=7aa887e1dc72e692703c2af38a960d90/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:15,387 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,388 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,389 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,391 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,391 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,391 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:15,393 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:15,394 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7aa887e1dc72e692703c2af38a960d90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68089614, jitterRate=0.014614313840866089}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:15,395 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7aa887e1dc72e692703c2af38a960d90: Writing region info on filesystem at 1731558915381Initializing all the Stores at 1731558915383 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558915383Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558915383Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558915383Cleaning up temporary data from old regions at 1731558915391 (+8 ms)Region opened successfully at 1731558915395 (+4 ms) 2024-11-14T04:35:15,399 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731558915398/Put/seqid=0 2024-11-14T04:35:15,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741881_1058 (size=4826) 2024-11-14T04:35:15,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741881_1058 (size=4826) 2024-11-14T04:35:15,406 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41731/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 7aa887e1dc72e692703c2af38a960d90/a 2024-11-14T04:35:15,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741881_1058 (size=4826) 2024-11-14T04:35:15,414 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-14T04:35:15,414 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T04:35:15,414 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7aa887e1dc72e692703c2af38a960d90: 2024-11-14T04:35:15,416 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/a/7c7af9c34fa0491798b0b8c478055a5d_SeqId_3_ 2024-11-14T04:35:15,416 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41731/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 7aa887e1dc72e692703c2af38a960d90/a as hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/a/7c7af9c34fa0491798b0b8c478055a5d_SeqId_3_ - updating store file list. 2024-11-14T04:35:15,422 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7c7af9c34fa0491798b0b8c478055a5d_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-14T04:35:15,423 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/a/7c7af9c34fa0491798b0b8c478055a5d_SeqId_3_ into 7aa887e1dc72e692703c2af38a960d90/a 2024-11-14T04:35:15,423 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41731/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 7aa887e1dc72e692703c2af38a960d90/a (new location: hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/a/7c7af9c34fa0491798b0b8c478055a5d_SeqId_3_) 2024-11-14T04:35:15,461 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337, size=0 (0bytes) 2024-11-14T04:35:15,461 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 might be still open, length is 0 2024-11-14T04:35:15,461 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 2024-11-14T04:35:15,462 WARN [IPC Server handler 3 on default port 41731 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-14T04:35:15,462 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 after 0ms 2024-11-14T04:35:17,562 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:49948 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49948 dst: /127.0.0.1:33533 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33533 remote=/127.0.0.1:49948]. Total timeout mills is 60000, 57867 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:17,563 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:47972 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47972 dst: /127.0.0.1:36191 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:17,563 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:41562 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37377:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41562 dst: /127.0.0.1:37377 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:17,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741880_1059 (size=606) 2024-11-14T04:35:17,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741880_1059 (size=606) 2024-11-14T04:35:19,464 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 after 4002ms 2024-11-14T04:35:19,498 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:19,498 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 took 4037ms 2024-11-14T04:35:19,500 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337; continuing. 2024-11-14T04:35:19,500 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 so closing down 2024-11-14T04:35:19,500 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:19,502 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731558915337.temp 2024-11-14T04:35:19,527 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005-wal.1731558915337.temp 2024-11-14T04:35:19,528 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:19,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741882_1060 (size=329) 2024-11-14T04:35:19,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741882_1060 (size=329) 2024-11-14T04:35:19,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741882_1060 (size=329) 2024-11-14T04:35:19,535 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005-wal.1731558915337.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:19,536 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005-wal.1731558915337.temp to hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005 2024-11-14T04:35:19,536 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T04:35:19,536 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337, journal: Splitting hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337, size=0 (0bytes) at 1731558915461Finishing writing output for hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 so closing down at 1731558919500 (+4039 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005-wal.1731558915337.temp at 1731558919528 (+28 ms)3 split writer threads finished at 1731558919528Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005-wal.1731558915337.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731558919535 (+7 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005-wal.1731558915337.temp to hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005 at 1731558919536 (+1 ms)Processed 2 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337, size=0, length=0, corrupted=false, cancelled=false at 1731558919536 2024-11-14T04:35:19,537 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 to hdfs://localhost:41731/hbase/oldWALs/wal.1731558915337 2024-11-14T04:35:19,538 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005 2024-11-14T04:35:19,538 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:19,540 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:19,553 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558919540, exclude list is [], retry=0 2024-11-14T04:35:19,556 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:19,556 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:19,557 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:19,582 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558919540 2024-11-14T04:35:19,582 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:19,582 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7aa887e1dc72e692703c2af38a960d90, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:19,583 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:19,583 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,583 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,584 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,585 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa887e1dc72e692703c2af38a960d90 columnFamilyName a 2024-11-14T04:35:19,585 DEBUG [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:19,592 DEBUG [StoreFileOpener-7aa887e1dc72e692703c2af38a960d90-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 7c7af9c34fa0491798b0b8c478055a5d_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-14T04:35:19,592 DEBUG [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/a/7c7af9c34fa0491798b0b8c478055a5d_SeqId_3_ 2024-11-14T04:35:19,592 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(327): Store=7aa887e1dc72e692703c2af38a960d90/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:19,593 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,594 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa887e1dc72e692703c2af38a960d90 columnFamilyName b 2024-11-14T04:35:19,594 DEBUG [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:19,594 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(327): Store=7aa887e1dc72e692703c2af38a960d90/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:19,594 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,595 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7aa887e1dc72e692703c2af38a960d90 columnFamilyName c 2024-11-14T04:35:19,595 DEBUG [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:19,595 INFO [StoreOpener-7aa887e1dc72e692703c2af38a960d90-1 {}] regionserver.HStore(327): Store=7aa887e1dc72e692703c2af38a960d90/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:19,596 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,596 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,598 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,599 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005 2024-11-14T04:35:19,624 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:19,625 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005 2024-11-14T04:35:19,625 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7aa887e1dc72e692703c2af38a960d90 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-14T04:35:19,640 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/.tmp/a/4149e6d2af5a43c58dfb42590e0f4c67 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731558915428/Put/seqid=0 2024-11-14T04:35:19,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741884_1062 (size=5149) 2024-11-14T04:35:19,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741884_1062 (size=5149) 2024-11-14T04:35:19,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741884_1062 (size=5149) 2024-11-14T04:35:19,647 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/.tmp/a/4149e6d2af5a43c58dfb42590e0f4c67 2024-11-14T04:35:19,653 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/.tmp/a/4149e6d2af5a43c58dfb42590e0f4c67 as hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/a/4149e6d2af5a43c58dfb42590e0f4c67 2024-11-14T04:35:19,658 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/a/4149e6d2af5a43c58dfb42590e0f4c67, entries=1, sequenceid=5, filesize=5.0 K 2024-11-14T04:35:19,659 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 7aa887e1dc72e692703c2af38a960d90 in 34ms, sequenceid=5, compaction requested=false; wal=null 2024-11-14T04:35:19,659 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/0000000000000000005 2024-11-14T04:35:19,660 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,661 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,661 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:19,663 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7aa887e1dc72e692703c2af38a960d90 2024-11-14T04:35:19,665 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/7aa887e1dc72e692703c2af38a960d90/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-14T04:35:19,666 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7aa887e1dc72e692703c2af38a960d90; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67785482, jitterRate=0.010082393884658813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:19,666 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7aa887e1dc72e692703c2af38a960d90: Writing region info on filesystem at 1731558919583Initializing all the Stores at 1731558919584 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558919584Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558919584Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558919584Obtaining lock to block concurrent updates at 1731558919625 (+41 ms)Preparing flush snapshotting stores in 7aa887e1dc72e692703c2af38a960d90 at 1731558919625Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731558919626 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. at 1731558919626Flushing 7aa887e1dc72e692703c2af38a960d90/a: creating writer at 1731558919626Flushing 7aa887e1dc72e692703c2af38a960d90/a: appending metadata at 1731558919639 (+13 ms)Flushing 7aa887e1dc72e692703c2af38a960d90/a: closing flushed file at 1731558919639Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eac23c0: reopening flushed file at 1731558919652 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 7aa887e1dc72e692703c2af38a960d90 in 34ms, sequenceid=5, compaction requested=false; wal=null at 1731558919659 (+7 ms)Cleaning up temporary data from old regions at 1731558919661 (+2 ms)Region opened successfully at 1731558919666 (+5 ms) 2024-11-14T04:35:19,670 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7aa887e1dc72e692703c2af38a960d90, disabling compactions & flushes 2024-11-14T04:35:19,670 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. 2024-11-14T04:35:19,670 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. 2024-11-14T04:35:19,670 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. after waiting 0 ms 2024-11-14T04:35:19,670 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. 2024-11-14T04:35:19,671 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731558915229.7aa887e1dc72e692703c2af38a960d90. 2024-11-14T04:35:19,671 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7aa887e1dc72e692703c2af38a960d90: Waiting for close lock at 1731558919670Disabling compacts and flushes for region at 1731558919670Disabling writes for close at 1731558919670Writing region close event to WAL at 1731558919671 (+1 ms)Closed at 1731558919671 2024-11-14T04:35:19,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741883_1061 (size=175) 2024-11-14T04:35:19,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741883_1061 (size=175) 2024-11-14T04:35:19,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741883_1061 (size=175) 2024-11-14T04:35:19,676 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:19,676 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731558919540) 2024-11-14T04:35:19,688 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=415 (was 409) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:41731 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43583 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_282548805_22 at /127.0.0.1:48008 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:41731 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_282548805_22 at /127.0.0.1:49974 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43583 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=408 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=6756 (was 6758) 2024-11-14T04:35:19,700 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=415, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=408, ProcessCount=11, AvailableMemoryMB=6756 2024-11-14T04:35:19,716 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:19,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T04:35:19,723 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b024da739ae8,36617,1731558903563 2024-11-14T04:35:19,725 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e3dd1c3 2024-11-14T04:35:19,726 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T04:35:19,728 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T04:35:19,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T04:35:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-14T04:35:19,740 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T04:35:19,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-14T04:35:19,743 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:19,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T04:35:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:35:19,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741885_1063 (size=694) 2024-11-14T04:35:19,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741885_1063 (size=694) 2024-11-14T04:35:19,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741885_1063 (size=694) 2024-11-14T04:35:19,760 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 34ae018529a53a855c7af76d8741b2e3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc 2024-11-14T04:35:19,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741886_1064 (size=77) 2024-11-14T04:35:19,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741886_1064 (size=77) 2024-11-14T04:35:19,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741886_1064 (size=77) 2024-11-14T04:35:19,770 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:19,770 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 34ae018529a53a855c7af76d8741b2e3, disabling compactions & flushes 2024-11-14T04:35:19,770 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:19,770 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:19,770 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. after waiting 0 ms 2024-11-14T04:35:19,770 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:19,770 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:19,770 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 34ae018529a53a855c7af76d8741b2e3: Waiting for close lock at 1731558919770Disabling compacts and flushes for region at 1731558919770Disabling writes for close at 1731558919770Writing region close event to WAL at 1731558919770Closed at 1731558919770 2024-11-14T04:35:19,772 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T04:35:19,776 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731558919772"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731558919772"}]},"ts":"1731558919772"} 2024-11-14T04:35:19,780 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T04:35:19,781 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T04:35:19,784 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731558919781"}]},"ts":"1731558919781"} 2024-11-14T04:35:19,787 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-14T04:35:19,788 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {b024da739ae8=0} racks are {/default-rack=0} 2024-11-14T04:35:19,789 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T04:35:19,789 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T04:35:19,790 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T04:35:19,790 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T04:35:19,790 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T04:35:19,790 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T04:35:19,790 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T04:35:19,790 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T04:35:19,790 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T04:35:19,790 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T04:35:19,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, ASSIGN}] 2024-11-14T04:35:19,793 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, ASSIGN 2024-11-14T04:35:19,795 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, ASSIGN; state=OFFLINE, location=b024da739ae8,43439,1731558904285; forceNewPlan=false, retain=false 2024-11-14T04:35:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:35:19,947 INFO [b024da739ae8:36617 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T04:35:19,947 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=OPENING, regionLocation=b024da739ae8,43439,1731558904285 2024-11-14T04:35:19,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, ASSIGN because future has completed 2024-11-14T04:35:19,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,43439,1731558904285}] 2024-11-14T04:35:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:35:20,105 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T04:35:20,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33239, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T04:35:20,114 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,114 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 34ae018529a53a855c7af76d8741b2e3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:20,114 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,115 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:20,115 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,115 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,117 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,119 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf1 2024-11-14T04:35:20,119 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:20,120 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:20,120 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,122 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf2 2024-11-14T04:35:20,122 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:20,123 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:20,123 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,124 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,125 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,126 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,126 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,127 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T04:35:20,128 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,130 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:20,130 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 34ae018529a53a855c7af76d8741b2e3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65329669, jitterRate=-0.02651207149028778}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T04:35:20,130 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,131 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 34ae018529a53a855c7af76d8741b2e3: Running coprocessor pre-open hook at 1731558920115Writing region info on filesystem at 1731558920115Initializing all the Stores at 1731558920116 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558920116Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558920117 (+1 ms)Cleaning up temporary data from old regions at 1731558920126 (+9 ms)Running coprocessor post-open hooks at 1731558920130 (+4 ms)Region opened successfully at 1731558920131 (+1 ms) 2024-11-14T04:35:20,133 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., pid=6, masterSystemTime=1731558920105 2024-11-14T04:35:20,135 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,135 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,136 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=OPEN, openSeqNum=2, regionLocation=b024da739ae8,43439,1731558904285 2024-11-14T04:35:20,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,43439,1731558904285 because future has completed 2024-11-14T04:35:20,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T04:35:20,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,43439,1731558904285 in 189 msec 2024-11-14T04:35:20,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T04:35:20,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, ASSIGN in 353 msec 2024-11-14T04:35:20,148 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T04:35:20,148 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731558920148"}]},"ts":"1731558920148"} 2024-11-14T04:35:20,150 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-14T04:35:20,151 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T04:35:20,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 418 msec 2024-11-14T04:35:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T04:35:20,376 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-14T04:35:20,376 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-14T04:35:20,380 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T04:35:20,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-14T04:35:20,388 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T04:35:20,388 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-14T04:35:20,402 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=2] 2024-11-14T04:35:20,403 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T04:35:20,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T04:35:20,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=34ae018529a53a855c7af76d8741b2e3, source=b024da739ae8,43439,1731558904285, destination=b024da739ae8,42805,1731558904393, warming up region on b024da739ae8,42805,1731558904393 2024-11-14T04:35:20,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=34ae018529a53a855c7af76d8741b2e3, source=b024da739ae8,43439,1731558904285, destination=b024da739ae8,42805,1731558904393, running balancer 2024-11-14T04:35:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE 2024-11-14T04:35:20,424 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE 2024-11-14T04:35:20,427 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=CLOSING, regionLocation=b024da739ae8,43439,1731558904285 2024-11-14T04:35:20,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] regionserver.HRegion(7855): Warmup {ENCODED => 34ae018529a53a855c7af76d8741b2e3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:20,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:20,428 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,430 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf1 2024-11-14T04:35:20,430 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:20,430 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:20,431 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE because future has completed 2024-11-14T04:35:20,432 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf2 2024-11-14T04:35:20,432 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:20,432 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:20,432 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T04:35:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] regionserver.HRegion(1722): Closing 34ae018529a53a855c7af76d8741b2e3, disabling compactions & flushes 2024-11-14T04:35:20,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. after waiting 0 ms 2024-11-14T04:35:20,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,43439,1731558904285}] 2024-11-14T04:35:20,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] regionserver.HRegion(1676): Region close journal for 34ae018529a53a855c7af76d8741b2e3: Waiting for close lock at 1731558920432Disabling compacts and flushes for region at 1731558920432Disabling writes for close at 1731558920433 (+1 ms)Writing region close event to WAL at 1731558920436 (+3 ms)Closed at 1731558920436 2024-11-14T04:35:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-14T04:35:20,593 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,593 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T04:35:20,594 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 34ae018529a53a855c7af76d8741b2e3, disabling compactions & flushes 2024-11-14T04:35:20,594 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,594 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,595 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. after waiting 0 ms 2024-11-14T04:35:20,595 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,595 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 34ae018529a53a855c7af76d8741b2e3 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-14T04:35:20,615 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf1/cb053c81677d4408aed429479fd2c1f3 is 35, key is r1/cf1:q/1731558920406/Put/seqid=0 2024-11-14T04:35:20,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741887_1065 (size=4783) 2024-11-14T04:35:20,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741887_1065 (size=4783) 2024-11-14T04:35:20,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741887_1065 (size=4783) 2024-11-14T04:35:20,623 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf1/cb053c81677d4408aed429479fd2c1f3 2024-11-14T04:35:20,631 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf1/cb053c81677d4408aed429479fd2c1f3 as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/cb053c81677d4408aed429479fd2c1f3 2024-11-14T04:35:20,637 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/cb053c81677d4408aed429479fd2c1f3, entries=1, sequenceid=5, filesize=4.7 K 2024-11-14T04:35:20,638 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 34ae018529a53a855c7af76d8741b2e3 in 43ms, sequenceid=5, compaction requested=false 2024-11-14T04:35:20,639 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-14T04:35:20,644 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-14T04:35:20,647 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,647 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 34ae018529a53a855c7af76d8741b2e3: Waiting for close lock at 1731558920594Running coprocessor pre-close hooks at 1731558920594Disabling compacts and flushes for region at 1731558920594Disabling writes for close at 1731558920595 (+1 ms)Obtaining lock to block concurrent updates at 1731558920595Preparing flush snapshotting stores in 34ae018529a53a855c7af76d8741b2e3 at 1731558920595Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731558920595Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. at 1731558920597 (+2 ms)Flushing 34ae018529a53a855c7af76d8741b2e3/cf1: creating writer at 1731558920597Flushing 34ae018529a53a855c7af76d8741b2e3/cf1: appending metadata at 1731558920614 (+17 ms)Flushing 34ae018529a53a855c7af76d8741b2e3/cf1: closing flushed file at 1731558920615 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ca7c6cd: reopening flushed file at 1731558920630 (+15 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 34ae018529a53a855c7af76d8741b2e3 in 43ms, sequenceid=5, compaction requested=false at 1731558920639 (+9 ms)Writing region close event to WAL at 1731558920640 (+1 ms)Running coprocessor post-close hooks at 1731558920645 (+5 ms)Closed at 1731558920647 (+2 ms) 2024-11-14T04:35:20,647 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 34ae018529a53a855c7af76d8741b2e3 move to b024da739ae8,42805,1731558904393 record at close sequenceid=5 2024-11-14T04:35:20,651 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,651 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=CLOSED 2024-11-14T04:35:20,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,43439,1731558904285 because future has completed 2024-11-14T04:35:20,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T04:35:20,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,43439,1731558904285 in 222 msec 2024-11-14T04:35:20,660 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE; state=CLOSED, location=b024da739ae8,42805,1731558904393; forceNewPlan=false, retain=false 2024-11-14T04:35:20,811 INFO [b024da739ae8:36617 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T04:35:20,811 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=OPENING, regionLocation=b024da739ae8,42805,1731558904393 2024-11-14T04:35:20,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE because future has completed 2024-11-14T04:35:20,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,42805,1731558904393}] 2024-11-14T04:35:20,973 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,973 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 34ae018529a53a855c7af76d8741b2e3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:20,973 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,973 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:20,974 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,974 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,975 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,976 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf1 2024-11-14T04:35:20,976 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:20,983 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/cb053c81677d4408aed429479fd2c1f3 2024-11-14T04:35:20,983 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:20,983 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,984 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf2 2024-11-14T04:35:20,984 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:20,985 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:20,985 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,986 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,987 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,988 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,988 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,989 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T04:35:20,990 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,991 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 34ae018529a53a855c7af76d8741b2e3; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70328241, jitterRate=0.047972455620765686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T04:35:20,991 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:20,992 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 34ae018529a53a855c7af76d8741b2e3: Running coprocessor pre-open hook at 1731558920974Writing region info on filesystem at 1731558920974Initializing all the Stores at 1731558920975 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558920975Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558920975Cleaning up temporary data from old regions at 1731558920988 (+13 ms)Running coprocessor post-open hooks at 1731558920991 (+3 ms)Region opened successfully at 1731558920992 (+1 ms) 2024-11-14T04:35:20,993 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., pid=9, masterSystemTime=1731558920968 2024-11-14T04:35:20,996 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,996 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:20,997 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=OPEN, openSeqNum=9, regionLocation=b024da739ae8,42805,1731558904393 2024-11-14T04:35:21,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,42805,1731558904393 because future has completed 2024-11-14T04:35:21,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-14T04:35:21,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,42805,1731558904393 in 186 msec 2024-11-14T04:35:21,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE in 582 msec 2024-11-14T04:35:21,030 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T04:35:21,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T04:35:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:48144 deadline: 1731558981036, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b024da739ae8 port=42805 startCode=1731558904393. As of locationSeqNum=5. 2024-11-14T04:35:21,062 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b024da739ae8 port=42805 startCode=1731558904393. As of locationSeqNum=5. 2024-11-14T04:35:21,063 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b024da739ae8 port=42805 startCode=1731558904393. As of locationSeqNum=5. 2024-11-14T04:35:21,063 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,42805,1731558904393, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b024da739ae8 port=42805 startCode=1731558904393. As of locationSeqNum=5. 2024-11-14T04:35:21,181 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 34ae018529a53a855c7af76d8741b2e3 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-14T04:35:21,203 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf1/ae06a4a3abbd41feb3fc77e549a6fb65 is 29, key is r1/cf1:/1731558921173/DeleteFamily/seqid=0 2024-11-14T04:35:21,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741888_1066 (size=4906) 2024-11-14T04:35:21,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741888_1066 (size=4906) 2024-11-14T04:35:21,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741888_1066 (size=4906) 2024-11-14T04:35:21,216 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf1/ae06a4a3abbd41feb3fc77e549a6fb65 2024-11-14T04:35:21,223 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ae06a4a3abbd41feb3fc77e549a6fb65 2024-11-14T04:35:21,244 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf2/b59ca35bec14426c8184fcc6fb13fa79 is 29, key is r1/cf2:/1731558921173/DeleteFamily/seqid=0 2024-11-14T04:35:21,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741889_1067 (size=4906) 2024-11-14T04:35:21,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741889_1067 (size=4906) 2024-11-14T04:35:21,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741889_1067 (size=4906) 2024-11-14T04:35:21,254 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf2/b59ca35bec14426c8184fcc6fb13fa79 2024-11-14T04:35:21,260 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b59ca35bec14426c8184fcc6fb13fa79 2024-11-14T04:35:21,261 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf1/ae06a4a3abbd41feb3fc77e549a6fb65 as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/ae06a4a3abbd41feb3fc77e549a6fb65 2024-11-14T04:35:21,269 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ae06a4a3abbd41feb3fc77e549a6fb65 2024-11-14T04:35:21,269 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/ae06a4a3abbd41feb3fc77e549a6fb65, entries=1, sequenceid=12, filesize=4.8 K 2024-11-14T04:35:21,270 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf2/b59ca35bec14426c8184fcc6fb13fa79 as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/b59ca35bec14426c8184fcc6fb13fa79 2024-11-14T04:35:21,277 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b59ca35bec14426c8184fcc6fb13fa79 2024-11-14T04:35:21,278 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/b59ca35bec14426c8184fcc6fb13fa79, entries=1, sequenceid=12, filesize=4.8 K 2024-11-14T04:35:21,279 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 34ae018529a53a855c7af76d8741b2e3 in 98ms, sequenceid=12, compaction requested=false 2024-11-14T04:35:21,279 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 34ae018529a53a855c7af76d8741b2e3: 2024-11-14T04:35:21,282 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T04:35:21,284 DEBUG [Time-limited test {}] regionserver.HStore(1541): 34ae018529a53a855c7af76d8741b2e3/cf1 is initiating major compaction (all files) 2024-11-14T04:35:21,284 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T04:35:21,284 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:21,285 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 34ae018529a53a855c7af76d8741b2e3/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:21,285 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/cb053c81677d4408aed429479fd2c1f3, hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/ae06a4a3abbd41feb3fc77e549a6fb65] into tmpdir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp, totalSize=9.5 K 2024-11-14T04:35:21,287 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cb053c81677d4408aed429479fd2c1f3, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731558920406 2024-11-14T04:35:21,287 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ae06a4a3abbd41feb3fc77e549a6fb65, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-14T04:35:21,300 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 34ae018529a53a855c7af76d8741b2e3#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T04:35:21,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741890_1068 (size=4626) 2024-11-14T04:35:21,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741890_1068 (size=4626) 2024-11-14T04:35:21,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741890_1068 (size=4626) 2024-11-14T04:35:21,761 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf1/59ead0eb891042548df06c77ea065e41 as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/59ead0eb891042548df06c77ea065e41 2024-11-14T04:35:21,776 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 34ae018529a53a855c7af76d8741b2e3/cf1 of 34ae018529a53a855c7af76d8741b2e3 into 59ead0eb891042548df06c77ea065e41(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T04:35:21,776 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 34ae018529a53a855c7af76d8741b2e3: 2024-11-14T04:35:21,776 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-14T04:35:21,776 DEBUG [Time-limited test {}] regionserver.HStore(1541): 34ae018529a53a855c7af76d8741b2e3/cf2 is initiating major compaction (all files) 2024-11-14T04:35:21,776 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T04:35:21,776 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T04:35:21,776 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 34ae018529a53a855c7af76d8741b2e3/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:21,776 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/b59ca35bec14426c8184fcc6fb13fa79] into tmpdir=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp, totalSize=4.8 K 2024-11-14T04:35:21,777 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b59ca35bec14426c8184fcc6fb13fa79, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-14T04:35:21,783 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 34ae018529a53a855c7af76d8741b2e3#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T04:35:21,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741891_1069 (size=4592) 2024-11-14T04:35:21,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741891_1069 (size=4592) 2024-11-14T04:35:21,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741891_1069 (size=4592) 2024-11-14T04:35:21,796 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/.tmp/cf2/65bd7a7449e545b795466fe79f3e5a5e as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/65bd7a7449e545b795466fe79f3e5a5e 2024-11-14T04:35:21,803 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 34ae018529a53a855c7af76d8741b2e3/cf2 of 34ae018529a53a855c7af76d8741b2e3 into 65bd7a7449e545b795466fe79f3e5a5e(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T04:35:21,803 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 34ae018529a53a855c7af76d8741b2e3: 2024-11-14T04:35:21,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=34ae018529a53a855c7af76d8741b2e3, source=b024da739ae8,42805,1731558904393, destination=b024da739ae8,43439,1731558904285, warming up region on b024da739ae8,43439,1731558904285 2024-11-14T04:35:21,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=34ae018529a53a855c7af76d8741b2e3, source=b024da739ae8,42805,1731558904393, destination=b024da739ae8,43439,1731558904285, running balancer 2024-11-14T04:35:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE 2024-11-14T04:35:21,810 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE 2024-11-14T04:35:21,812 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=CLOSING, regionLocation=b024da739ae8,42805,1731558904393 2024-11-14T04:35:21,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] regionserver.HRegion(7855): Warmup {ENCODED => 34ae018529a53a855c7af76d8741b2e3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:21,814 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:21,815 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf1 2024-11-14T04:35:21,816 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:21,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE because future has completed 2024-11-14T04:35:21,816 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T04:35:21,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,42805,1731558904393}] 2024-11-14T04:35:21,824 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/59ead0eb891042548df06c77ea065e41 2024-11-14T04:35:21,829 INFO [StoreFileOpener-34ae018529a53a855c7af76d8741b2e3-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ae06a4a3abbd41feb3fc77e549a6fb65 2024-11-14T04:35:21,829 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/ae06a4a3abbd41feb3fc77e549a6fb65 2024-11-14T04:35:21,835 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/cb053c81677d4408aed429479fd2c1f3 2024-11-14T04:35:21,835 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:21,835 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:21,836 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf2 2024-11-14T04:35:21,836 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:21,843 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/65bd7a7449e545b795466fe79f3e5a5e 2024-11-14T04:35:21,849 INFO [StoreFileOpener-34ae018529a53a855c7af76d8741b2e3-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b59ca35bec14426c8184fcc6fb13fa79 2024-11-14T04:35:21,849 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/b59ca35bec14426c8184fcc6fb13fa79 2024-11-14T04:35:21,849 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] regionserver.HRegion(1722): Closing 34ae018529a53a855c7af76d8741b2e3, disabling compactions & flushes 2024-11-14T04:35:21,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. after waiting 0 ms 2024-11-14T04:35:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:21,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43439 {}] regionserver.HRegion(1676): Region close journal for 34ae018529a53a855c7af76d8741b2e3: Waiting for close lock at 1731558921849Disabling compacts and flushes for region at 1731558921849Disabling writes for close at 1731558921849Writing region close event to WAL at 1731558921851 (+2 ms)Closed at 1731558921851 2024-11-14T04:35:21,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-14T04:35:21,971 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:21,971 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T04:35:21,971 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 34ae018529a53a855c7af76d8741b2e3, disabling compactions & flushes 2024-11-14T04:35:21,971 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:21,971 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:21,971 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. after waiting 0 ms 2024-11-14T04:35:21,971 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:21,972 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/cb053c81677d4408aed429479fd2c1f3, hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/ae06a4a3abbd41feb3fc77e549a6fb65] to archive 2024-11-14T04:35:21,974 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T04:35:21,979 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/cb053c81677d4408aed429479fd2c1f3 to hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/cb053c81677d4408aed429479fd2c1f3 2024-11-14T04:35:21,981 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/ae06a4a3abbd41feb3fc77e549a6fb65 to hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/ae06a4a3abbd41feb3fc77e549a6fb65 2024-11-14T04:35:21,994 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/b59ca35bec14426c8184fcc6fb13fa79] to archive 2024-11-14T04:35:21,995 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T04:35:21,997 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/b59ca35bec14426c8184fcc6fb13fa79 to hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/b59ca35bec14426c8184fcc6fb13fa79 2024-11-14T04:35:22,002 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-14T04:35:22,002 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:22,003 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 34ae018529a53a855c7af76d8741b2e3: Waiting for close lock at 1731558921971Running coprocessor pre-close hooks at 1731558921971Disabling compacts and flushes for region at 1731558921971Disabling writes for close at 1731558921971Writing region close event to WAL at 1731558921998 (+27 ms)Running coprocessor post-close hooks at 1731558922002 (+4 ms)Closed at 1731558922002 2024-11-14T04:35:22,003 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 34ae018529a53a855c7af76d8741b2e3 move to b024da739ae8,43439,1731558904285 record at close sequenceid=12 2024-11-14T04:35:22,005 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,006 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=CLOSED 2024-11-14T04:35:22,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,42805,1731558904393 because future has completed 2024-11-14T04:35:22,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-14T04:35:22,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,42805,1731558904393 in 195 msec 2024-11-14T04:35:22,014 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE; state=CLOSED, location=b024da739ae8,43439,1731558904285; forceNewPlan=false, retain=false 2024-11-14T04:35:22,165 INFO [b024da739ae8:36617 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T04:35:22,165 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=OPENING, regionLocation=b024da739ae8,43439,1731558904285 2024-11-14T04:35:22,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE because future has completed 2024-11-14T04:35:22,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,43439,1731558904285}] 2024-11-14T04:35:22,329 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:22,329 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 34ae018529a53a855c7af76d8741b2e3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:22,330 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,330 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:22,330 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,330 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,333 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,334 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf1 2024-11-14T04:35:22,334 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:22,343 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/59ead0eb891042548df06c77ea065e41 2024-11-14T04:35:22,343 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:22,343 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,344 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf2 2024-11-14T04:35:22,344 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:22,351 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/65bd7a7449e545b795466fe79f3e5a5e 2024-11-14T04:35:22,351 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:22,351 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,353 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,354 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,355 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,355 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,356 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T04:35:22,359 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,360 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 34ae018529a53a855c7af76d8741b2e3; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64990973, jitterRate=-0.03155903518199921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T04:35:22,360 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,361 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 34ae018529a53a855c7af76d8741b2e3: Running coprocessor pre-open hook at 1731558922331Writing region info on filesystem at 1731558922331Initializing all the Stores at 1731558922332 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558922332Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558922332Cleaning up temporary data from old regions at 1731558922355 (+23 ms)Running coprocessor post-open hooks at 1731558922360 (+5 ms)Region opened successfully at 1731558922361 (+1 ms) 2024-11-14T04:35:22,362 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., pid=12, masterSystemTime=1731558922320 2024-11-14T04:35:22,365 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:22,365 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:22,366 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=OPEN, openSeqNum=18, regionLocation=b024da739ae8,43439,1731558904285 2024-11-14T04:35:22,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,43439,1731558904285 because future has completed 2024-11-14T04:35:22,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-14T04:35:22,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,43439,1731558904285 in 202 msec 2024-11-14T04:35:22,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, REOPEN/MOVE in 564 msec 2024-11-14T04:35:22,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T04:35:22,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T04:35:22,417 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b024da739ae8,43439,1731558904285: testing ***** 2024-11-14T04:35:22,417 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-14T04:35:22,420 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-14T04:35:22,422 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-14T04:35:22,426 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-14T04:35:22,428 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-14T04:35:22,438 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 474069640 }, "NonHeapMemoryUsage": { "committed": 171638784, "init": 7667712, "max": -1, "used": 169019072 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b024da739ae8", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 42, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 14, "ProcessCallTime_98th_percentile": 42, "ProcessCallTime_99th_percentile": 42, "ProcessCallTime_99.9th_percentile": 42, "ProcessCallTime_TimeRangeCount_0-1": 11, "ProcessCallTime_TimeRangeCount_1-3": 14, "ProcessCallTime_TimeRangeCount_3-10": 4, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 42, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 14, "TotalCallTime_98th_percentile": 42, "TotalCallTime_99th_percentile": 42, "TotalCallTime_99.9th_percentile": 42, "TotalCallTime_TimeRangeCount_0-1": 10, "TotalCallTime_TimeRangeCount_1-3": 15, "TotalCallTime_TimeRangeCount_3-10": 4, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b024da739ae8", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:51455", "tag.serverName": "b024da739ae8,43439,1731558904285", "tag.clusterId": "ead3ed9a-fa97-4c97-9e40-e4af243b7358", "tag.Context": "regionserver", "tag.Hostname": "b024da739ae8", "regionCount": 1, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731558904285, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.7697053155307054, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 4, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 1, "ScanTime_max": 1, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 1, "Put_min": 2, "Put_max": 2, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 2, "Put_98th_percentile": 2, "Put_99th_percentile": 2, "Put_99.9th_percentile": 2, "Put_TimeRangeCount_1-3": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 43, "FlushTime_max": 43, "FlushTime_mean": 43, "FlushTime_25th_percentile": 43, "FlushTime_median": 43, "FlushTime_75th_percentile": 43, "FlushTime_90th_percentile": 43, "FlushTime_95th_percentile": 43, "FlushTime_98th_percentile": 43, "FlushTime_99th_percentile": 43, "FlushTime_99.9th_percentile": 43, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 64, "ScanSize_max": 64, "ScanSize_mean": 64, "ScanSize_25th_percentile": 64, "ScanSize_median": 64, "ScanSize_75th_percentile": 64, "ScanSize_90th_percentile": 64, "ScanSize_95th_percentile": 64, "ScanSize_98th_percentile": 64, "ScanSize_99th_percentile": 64, "ScanSize_99.9th_percentile": 64, "ScanSize_SizeRangeCount_10-100": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-14T04:35:22,440 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36617 {}] master.MasterRpcServices(700): b024da739ae8,43439,1731558904285 reported a fatal error: ***** ABORTING region server b024da739ae8,43439,1731558904285: testing ***** 2024-11-14T04:35:22,442 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b024da739ae8,43439,1731558904285' ***** 2024-11-14T04:35:22,443 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-14T04:35:22,443 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T04:35:22,443 INFO [RS:0;b024da739ae8:43439 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-14T04:35:22,443 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T04:35:22,443 INFO [RS:0;b024da739ae8:43439 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-14T04:35:22,443 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(3091): Received CLOSE for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:37300 deadline: 1731558982443, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b024da739ae8 port=43439 startCode=1731558904285. As of locationSeqNum=12. 2024-11-14T04:35:22,444 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(956): aborting server b024da739ae8,43439,1731558904285 2024-11-14T04:35:22,444 INFO [RS:0;b024da739ae8:43439 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:35:22,444 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 34ae018529a53a855c7af76d8741b2e3, disabling compactions & flushes 2024-11-14T04:35:22,444 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:22,444 INFO [RS:0;b024da739ae8:43439 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b024da739ae8:43439. 2024-11-14T04:35:22,444 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:22,444 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. after waiting 0 ms 2024-11-14T04:35:22,444 DEBUG [RS:0;b024da739ae8:43439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:35:22,444 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:22,444 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,42805,1731558904393, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,42805,1731558904393, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b024da739ae8 port=43439 startCode=1731558904285. As of locationSeqNum=12. 2024-11-14T04:35:22,444 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,42805,1731558904393, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b024da739ae8 port=43439 startCode=1731558904285. As of locationSeqNum=12. 2024-11-14T04:35:22,445 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,42805,1731558904393, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b024da739ae8 port=43439 startCode=1731558904285. As of locationSeqNum=12. 2024-11-14T04:35:22,445 DEBUG [RS:0;b024da739ae8:43439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:35:22,445 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T04:35:22,445 DEBUG [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(1325): Online Regions={34ae018529a53a855c7af76d8741b2e3=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.} 2024-11-14T04:35:22,446 DEBUG [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(1351): Waiting on 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:22,446 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:22,446 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 34ae018529a53a855c7af76d8741b2e3: Waiting for close lock at 1731558922444Running coprocessor pre-close hooks at 1731558922444Disabling compacts and flushes for region at 1731558922444Disabling writes for close at 1731558922444Writing region close event to WAL at 1731558922446 (+2 ms)Running coprocessor post-close hooks at 1731558922446Closed at 1731558922446 2024-11-14T04:35:22,446 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:22,495 INFO [regionserver/b024da739ae8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:35:22,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b024da739ae8,43439,1731558904285 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:22,554 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b024da739ae8,43439,1731558904285 aborting 2024-11-14T04:35:22,554 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b024da739ae8,43439,1731558904285 aborting 2024-11-14T04:35:22,555 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=12 from cache 2024-11-14T04:35:22,646 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(976): stopping server b024da739ae8,43439,1731558904285; all regions closed. 2024-11-14T04:35:22,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741834_1010 (size=2288) 2024-11-14T04:35:22,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741834_1010 (size=2288) 2024-11-14T04:35:22,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741834_1010 (size=2288) 2024-11-14T04:35:22,656 DEBUG [RS:0;b024da739ae8:43439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:35:22,656 INFO [RS:0;b024da739ae8:43439 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:35:22,657 INFO [RS:0;b024da739ae8:43439 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:35:22,657 INFO [RS:0;b024da739ae8:43439 {}] hbase.ChoreService(370): Chore service for: regionserver/b024da739ae8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T04:35:22,657 INFO [regionserver/b024da739ae8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:35:22,657 INFO [RS:0;b024da739ae8:43439 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T04:35:22,657 INFO [RS:0;b024da739ae8:43439 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T04:35:22,658 INFO [RS:0;b024da739ae8:43439 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T04:35:22,658 INFO [RS:0;b024da739ae8:43439 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:35:22,658 INFO [RS:0;b024da739ae8:43439 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43439 2024-11-14T04:35:22,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b024da739ae8,43439,1731558904285 2024-11-14T04:35:22,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:35:22,723 INFO [RS:0;b024da739ae8:43439 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:35:22,734 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b024da739ae8,43439,1731558904285] 2024-11-14T04:35:22,743 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b024da739ae8,43439,1731558904285 already deleted, retry=false 2024-11-14T04:35:22,744 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of b024da739ae8,43439,1731558904285 on b024da739ae8,36617,1731558903563 2024-11-14T04:35:22,751 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure b024da739ae8,43439,1731558904285, splitWal=true, meta=false 2024-11-14T04:35:22,755 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for b024da739ae8,43439,1731558904285 (carryingMeta=false) b024da739ae8,43439,1731558904285/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@381b52d0[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T04:35:22,756 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure b024da739ae8,43439,1731558904285, splitWal=true, meta=false 2024-11-14T04:35:22,758 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): b024da739ae8,43439,1731558904285 had 1 regions 2024-11-14T04:35:22,759 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure b024da739ae8,43439,1731558904285, splitWal=true, meta=false, isMeta: false 2024-11-14T04:35:22,760 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting 2024-11-14T04:35:22,762 INFO [PEWorker-1 {}] master.SplitWALManager(105): b024da739ae8,43439,1731558904285 WAL count=1, meta=false 2024-11-14T04:35:22,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b024da739ae8%2C43439%2C1731558904285.1731558905913}] 2024-11-14T04:35:22,765 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=18] 2024-11-14T04:35:22,767 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b024da739ae8:43439 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b024da739ae8/172.17.0.3:43439 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:22,767 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=18, error=java.net.ConnectException: Call to address=b024da739ae8:43439 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b024da739ae8/172.17.0.3:43439 2024-11-14T04:35:22,767 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-14T04:35:22,767 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=18 from cache 2024-11-14T04:35:22,768 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address b024da739ae8:43439 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b024da739ae8/172.17.0.3:43439 2024-11-14T04:35:22,770 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=b024da739ae8,42805,1731558904393 2024-11-14T04:35:22,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b024da739ae8%2C43439%2C1731558904285.1731558905913, worker=b024da739ae8,42805,1731558904393}] 2024-11-14T04:35:22,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:35:22,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43439-0x10137a0072b0001, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:35:22,834 INFO [RS:0;b024da739ae8:43439 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:35:22,834 INFO [RS:0;b024da739ae8:43439 {}] regionserver.HRegionServer(1031): Exiting; stopping=b024da739ae8,43439,1731558904285; zookeeper connection closed. 2024-11-14T04:35:22,835 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7b156767 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7b156767 2024-11-14T04:35:22,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42805 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-14T04:35:22,955 INFO [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913, size=2.2 K (2288bytes) 2024-11-14T04:35:22,956 INFO [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913 2024-11-14T04:35:22,956 INFO [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913 after 0ms 2024-11-14T04:35:22,981 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:22,981 INFO [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913 took 26ms 2024-11-14T04:35:22,987 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 34ae018529a53a855c7af76d8741b2e3: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-14T04:35:22,988 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913 so closing down 2024-11-14T04:35:22,988 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:22,988 INFO [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:22,988 INFO [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-14T04:35:22,988 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913, journal: Splitting hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913, size=2.2 K (2288bytes) at 1731558922956Finishing writing output for hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913 so closing down at 1731558922988 (+32 ms)3 split writer threads finished at 1731558922988Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1731558922988 2024-11-14T04:35:22,988 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913 2024-11-14T04:35:22,989 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b024da739ae8:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-14T04:35:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36617 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-14T04:35:22,995 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting/b024da739ae8%2C43439%2C1731558904285.1731558905913 to hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/oldWALs 2024-11-14T04:35:22,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-14T04:35:22,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b024da739ae8%2C43439%2C1731558904285.1731558905913, worker=b024da739ae8,42805,1731558904393 in 224 msec 2024-11-14T04:35:22,999 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=b024da739ae8,42805,1731558904393 2024-11-14T04:35:23,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T04:35:23,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure b024da739ae8%2C43439%2C1731558904285.1731558905913, worker=b024da739ae8,42805,1731558904393 in 237 msec 2024-11-14T04:35:23,004 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting dir is empty, no logs to split. 2024-11-14T04:35:23,004 INFO [PEWorker-1 {}] master.SplitWALManager(105): b024da739ae8,43439,1731558904285 WAL count=0, meta=false 2024-11-14T04:35:23,004 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if b024da739ae8,43439,1731558904285 WAL splitting is done? wals=0, meta=false 2024-11-14T04:35:23,006 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for b024da739ae8,43439,1731558904285 failed, ignore...File hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/WALs/b024da739ae8,43439,1731558904285-splitting does not exist. 2024-11-14T04:35:23,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, ASSIGN}] 2024-11-14T04:35:23,009 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, ASSIGN 2024-11-14T04:35:23,010 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T04:35:23,077 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=18] 2024-11-14T04:35:23,077 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to b024da739ae8:43439 this server is in the failed servers list 2024-11-14T04:35:23,078 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b024da739ae8:43439 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b024da739ae8:43439 2024-11-14T04:35:23,078 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b024da739ae8:43439 2024-11-14T04:35:23,078 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,43439,1731558904285, seqNum=18 from cache 2024-11-14T04:35:23,160 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(204): Hosts are {b024da739ae8=0} racks are {/default-rack=0} 2024-11-14T04:35:23,161 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T04:35:23,161 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T04:35:23,161 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T04:35:23,161 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T04:35:23,161 INFO [b024da739ae8:36617 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T04:35:23,161 INFO [b024da739ae8:36617 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T04:35:23,161 DEBUG [b024da739ae8:36617 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T04:35:23,162 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=OPENING, regionLocation=b024da739ae8,42805,1731558904393 2024-11-14T04:35:23,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, ASSIGN because future has completed 2024-11-14T04:35:23,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,42805,1731558904393}] 2024-11-14T04:35:23,328 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:23,329 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 34ae018529a53a855c7af76d8741b2e3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:23,329 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,330 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:23,330 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,330 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,332 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,333 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf1 2024-11-14T04:35:23,334 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:23,341 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf1/59ead0eb891042548df06c77ea065e41 2024-11-14T04:35:23,341 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:23,342 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,343 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34ae018529a53a855c7af76d8741b2e3 columnFamilyName cf2 2024-11-14T04:35:23,343 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:23,349 DEBUG [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/cf2/65bd7a7449e545b795466fe79f3e5a5e 2024-11-14T04:35:23,349 INFO [StoreOpener-34ae018529a53a855c7af76d8741b2e3-1 {}] regionserver.HStore(327): Store=34ae018529a53a855c7af76d8741b2e3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:23,349 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,350 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,352 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,352 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,352 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,353 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T04:35:23,354 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,355 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 34ae018529a53a855c7af76d8741b2e3; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65214804, jitterRate=-0.028223693370819092}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T04:35:23,356 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:23,356 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 34ae018529a53a855c7af76d8741b2e3: Running coprocessor pre-open hook at 1731558923330Writing region info on filesystem at 1731558923330Initializing all the Stores at 1731558923332 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558923332Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558923332Cleaning up temporary data from old regions at 1731558923352 (+20 ms)Running coprocessor post-open hooks at 1731558923356 (+4 ms)Region opened successfully at 1731558923356 2024-11-14T04:35:23,357 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., pid=17, masterSystemTime=1731558923321 2024-11-14T04:35:23,359 DEBUG [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:23,360 INFO [RS_OPEN_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:23,360 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=34ae018529a53a855c7af76d8741b2e3, regionState=OPEN, openSeqNum=18, regionLocation=b024da739ae8,42805,1731558904393 2024-11-14T04:35:23,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,42805,1731558904393 because future has completed 2024-11-14T04:35:23,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-14T04:35:23,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 34ae018529a53a855c7af76d8741b2e3, server=b024da739ae8,42805,1731558904393 in 195 msec 2024-11-14T04:35:23,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-14T04:35:23,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=34ae018529a53a855c7af76d8741b2e3, ASSIGN in 357 msec 2024-11-14T04:35:23,368 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server b024da739ae8,43439,1731558904285 after splitting done 2024-11-14T04:35:23,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure b024da739ae8,43439,1731558904285, splitWal=true, meta=false in 623 msec 2024-11-14T04:35:23,584 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., hostname=b024da739ae8,42805,1731558904393, seqNum=18] 2024-11-14T04:35:23,597 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=405 (was 415), OpenFileDescriptor=1043 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 408), ProcessCount=11 (was 11), AvailableMemoryMB=6709 (was 6756) 2024-11-14T04:35:23,599 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1043 is superior to 1024 2024-11-14T04:35:23,609 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=405, OpenFileDescriptor=1043, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=6709 2024-11-14T04:35:23,609 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1043 is superior to 1024 2024-11-14T04:35:23,623 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:23,625 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:23,626 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:23,629 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-81526405, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/hregion-81526405, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:23,641 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-81526405/hregion-81526405.1731558923629, exclude list is [], retry=0 2024-11-14T04:35:23,644 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:23,644 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:23,644 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:23,670 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-81526405/hregion-81526405.1731558923629 2024-11-14T04:35:23,670 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-14T04:35:23,670 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 41814b12fcfaad453a8c9184c10c8a26, NAME => 'testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/hbase 2024-11-14T04:35:23,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741893_1071 (size=67) 2024-11-14T04:35:23,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741893_1071 (size=67) 2024-11-14T04:35:23,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741893_1071 (size=67) 2024-11-14T04:35:23,679 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:23,680 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,682 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41814b12fcfaad453a8c9184c10c8a26 columnFamilyName a 2024-11-14T04:35:23,682 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:23,682 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(327): Store=41814b12fcfaad453a8c9184c10c8a26/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:23,682 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,683 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41814b12fcfaad453a8c9184c10c8a26 columnFamilyName b 2024-11-14T04:35:23,683 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:23,684 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(327): Store=41814b12fcfaad453a8c9184c10c8a26/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:23,684 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,685 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41814b12fcfaad453a8c9184c10c8a26 columnFamilyName c 2024-11-14T04:35:23,685 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:23,686 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(327): Store=41814b12fcfaad453a8c9184c10c8a26/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:23,686 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,687 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,687 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,688 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,688 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,688 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:23,689 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,691 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:23,692 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41814b12fcfaad453a8c9184c10c8a26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66185388, jitterRate=-0.013760864734649658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:23,692 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41814b12fcfaad453a8c9184c10c8a26: Writing region info on filesystem at 1731558923679Initializing all the Stores at 1731558923680 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558923680Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558923680Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558923680Cleaning up temporary data from old regions at 1731558923688 (+8 ms)Region opened successfully at 1731558923692 (+4 ms) 2024-11-14T04:35:23,692 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 41814b12fcfaad453a8c9184c10c8a26, disabling compactions & flushes 2024-11-14T04:35:23,692 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. 2024-11-14T04:35:23,692 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. 2024-11-14T04:35:23,692 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. after waiting 0 ms 2024-11-14T04:35:23,692 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. 2024-11-14T04:35:23,693 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. 2024-11-14T04:35:23,693 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 41814b12fcfaad453a8c9184c10c8a26: Waiting for close lock at 1731558923692Disabling compacts and flushes for region at 1731558923692Disabling writes for close at 1731558923692Writing region close event to WAL at 1731558923693 (+1 ms)Closed at 1731558923693 2024-11-14T04:35:23,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741892_1070 (size=175) 2024-11-14T04:35:23,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741892_1070 (size=175) 2024-11-14T04:35:23,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741892_1070 (size=175) 2024-11-14T04:35:23,698 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:23,698 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-81526405:(num 1731558923629) 2024-11-14T04:35:23,698 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:23,700 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:23,718 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700, exclude list is [], retry=0 2024-11-14T04:35:23,720 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:23,721 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:23,721 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:23,749 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700 2024-11-14T04:35:23,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:23,750 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 41814b12fcfaad453a8c9184c10c8a26, NAME => 'testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:23,750 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:23,750 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,750 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,752 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,752 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41814b12fcfaad453a8c9184c10c8a26 columnFamilyName a 2024-11-14T04:35:23,753 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:23,753 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(327): Store=41814b12fcfaad453a8c9184c10c8a26/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:23,753 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,754 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41814b12fcfaad453a8c9184c10c8a26 columnFamilyName b 2024-11-14T04:35:23,754 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:23,755 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(327): Store=41814b12fcfaad453a8c9184c10c8a26/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:23,755 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,755 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41814b12fcfaad453a8c9184c10c8a26 columnFamilyName c 2024-11-14T04:35:23,756 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:23,756 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(327): Store=41814b12fcfaad453a8c9184c10c8a26/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:23,756 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,757 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,758 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,759 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,759 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,760 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:23,761 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:23,761 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41814b12fcfaad453a8c9184c10c8a26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74976332, jitterRate=0.11723440885543823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:23,762 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41814b12fcfaad453a8c9184c10c8a26: Writing region info on filesystem at 1731558923750Initializing all the Stores at 1731558923751 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558923751Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558923751Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558923751Cleaning up temporary data from old regions at 1731558923759 (+8 ms)Region opened successfully at 1731558923762 (+3 ms) 2024-11-14T04:35:23,788 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 41814b12fcfaad453a8c9184c10c8a26 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-14T04:35:23,802 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/a/27d51f1a53a948eda51f5f56096ffd5a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731558923762/Put/seqid=0 2024-11-14T04:35:23,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741895_1073 (size=5958) 2024-11-14T04:35:23,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741895_1073 (size=5958) 2024-11-14T04:35:23,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741895_1073 (size=5958) 2024-11-14T04:35:23,809 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/a/27d51f1a53a948eda51f5f56096ffd5a 2024-11-14T04:35:23,828 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/b/5cc79bcb73c449c6afd3951158cab1b7 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731558923770/Put/seqid=0 2024-11-14T04:35:23,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741896_1074 (size=5958) 2024-11-14T04:35:23,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741896_1074 (size=5958) 2024-11-14T04:35:23,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741896_1074 (size=5958) 2024-11-14T04:35:23,834 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/b/5cc79bcb73c449c6afd3951158cab1b7 2024-11-14T04:35:23,853 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/c/469057bbdf724f12bdba16fbaabb5d66 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731558923778/Put/seqid=0 2024-11-14T04:35:23,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741897_1075 (size=5958) 2024-11-14T04:35:23,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741897_1075 (size=5958) 2024-11-14T04:35:23,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741897_1075 (size=5958) 2024-11-14T04:35:23,860 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/c/469057bbdf724f12bdba16fbaabb5d66 2024-11-14T04:35:23,867 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/a/27d51f1a53a948eda51f5f56096ffd5a as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/a/27d51f1a53a948eda51f5f56096ffd5a 2024-11-14T04:35:23,873 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/a/27d51f1a53a948eda51f5f56096ffd5a, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T04:35:23,874 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/b/5cc79bcb73c449c6afd3951158cab1b7 as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/b/5cc79bcb73c449c6afd3951158cab1b7 2024-11-14T04:35:23,881 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/b/5cc79bcb73c449c6afd3951158cab1b7, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T04:35:23,882 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/c/469057bbdf724f12bdba16fbaabb5d66 as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/c/469057bbdf724f12bdba16fbaabb5d66 2024-11-14T04:35:23,888 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/c/469057bbdf724f12bdba16fbaabb5d66, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T04:35:23,889 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 41814b12fcfaad453a8c9184c10c8a26 in 101ms, sequenceid=33, compaction requested=false 2024-11-14T04:35:23,889 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 41814b12fcfaad453a8c9184c10c8a26: 2024-11-14T04:35:23,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-14T04:35:23,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-14T04:35:23,889 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 41814b12fcfaad453a8c9184c10c8a26, disabling compactions & flushes 2024-11-14T04:35:23,889 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. 2024-11-14T04:35:23,889 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. 2024-11-14T04:35:23,889 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. after waiting 0 ms 2024-11-14T04:35:23,889 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. 2024-11-14T04:35:23,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-14T04:35:23,890 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-14T04:35:23,891 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-14T04:35:23,891 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-14T04:35:23,891 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-14T04:35:23,891 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-14T04:35:23,893 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. 2024-11-14T04:35:23,893 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 41814b12fcfaad453a8c9184c10c8a26: Waiting for close lock at 1731558923889Disabling compacts and flushes for region at 1731558923889Disabling writes for close at 1731558923889Writing region close event to WAL at 1731558923892 (+3 ms)Closed at 1731558923892 2024-11-14T04:35:23,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741894_1072 (size=6311) 2024-11-14T04:35:23,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741894_1072 (size=6311) 2024-11-14T04:35:23,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741894_1072 (size=6311) 2024-11-14T04:35:23,899 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/b/5cc79bcb73c449c6afd3951158cab1b7 to hdfs://localhost:41731/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/b/5cc79bcb73c449c6afd3951158cab1b7 2024-11-14T04:35:23,913 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700, size=6.2 K (6311bytes) 2024-11-14T04:35:23,913 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700 2024-11-14T04:35:23,914 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700 after 1ms 2024-11-14T04:35:23,937 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:23,937 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700 took 24ms 2024-11-14T04:35:23,939 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700 so closing down 2024-11-14T04:35:23,939 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:23,940 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731558923700.temp 2024-11-14T04:35:23,964 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000003-wal.1731558923700.temp 2024-11-14T04:35:23,965 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:23,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741898_1076 (size=5575) 2024-11-14T04:35:23,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741898_1076 (size=5575) 2024-11-14T04:35:23,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741898_1076 (size=5575) 2024-11-14T04:35:23,972 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000003-wal.1731558923700.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:23,973 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000003-wal.1731558923700.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000032 2024-11-14T04:35:23,974 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-14T04:35:23,974 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700, journal: Splitting hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700, size=6.2 K (6311bytes) at 1731558923913Finishing writing output for hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700 so closing down at 1731558923939 (+26 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000003-wal.1731558923700.temp at 1731558923964 (+25 ms)3 split writer threads finished at 1731558923965 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000003-wal.1731558923700.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731558923972 (+7 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000003-wal.1731558923700.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000032 at 1731558923973 (+1 ms)Processed 32 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1731558923974 (+1 ms) 2024-11-14T04:35:23,975 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923700 to hdfs://localhost:41731/hbase/oldWALs/wal.1731558923700 2024-11-14T04:35:23,976 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000032 2024-11-14T04:35:23,976 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:23,978 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:23,992 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923978, exclude list is [], retry=0 2024-11-14T04:35:23,994 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:23,995 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:23,995 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:24,020 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923978 2024-11-14T04:35:24,021 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:24,021 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 41814b12fcfaad453a8c9184c10c8a26, NAME => 'testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:24,021 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:24,021 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,021 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,022 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,023 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41814b12fcfaad453a8c9184c10c8a26 columnFamilyName a 2024-11-14T04:35:24,023 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:24,028 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/a/27d51f1a53a948eda51f5f56096ffd5a 2024-11-14T04:35:24,028 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(327): Store=41814b12fcfaad453a8c9184c10c8a26/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:24,028 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,030 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41814b12fcfaad453a8c9184c10c8a26 columnFamilyName b 2024-11-14T04:35:24,030 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:24,030 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(327): Store=41814b12fcfaad453a8c9184c10c8a26/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:24,030 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,031 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41814b12fcfaad453a8c9184c10c8a26 columnFamilyName c 2024-11-14T04:35:24,031 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:24,036 DEBUG [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/c/469057bbdf724f12bdba16fbaabb5d66 2024-11-14T04:35:24,037 INFO [StoreOpener-41814b12fcfaad453a8c9184c10c8a26-1 {}] regionserver.HStore(327): Store=41814b12fcfaad453a8c9184c10c8a26/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:24,037 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,037 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,039 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,040 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000032 2024-11-14T04:35:24,076 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:24,078 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000032 2024-11-14T04:35:24,078 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 41814b12fcfaad453a8c9184c10c8a26 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-14T04:35:24,096 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/b/2756528ee3b240e39bd403c8b1d49a74 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731558923770/Put/seqid=0 2024-11-14T04:35:24,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741900_1078 (size=5958) 2024-11-14T04:35:24,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741900_1078 (size=5958) 2024-11-14T04:35:24,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741900_1078 (size=5958) 2024-11-14T04:35:24,103 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/b/2756528ee3b240e39bd403c8b1d49a74 2024-11-14T04:35:24,108 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/.tmp/b/2756528ee3b240e39bd403c8b1d49a74 as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/b/2756528ee3b240e39bd403c8b1d49a74 2024-11-14T04:35:24,114 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/b/2756528ee3b240e39bd403c8b1d49a74, entries=10, sequenceid=32, filesize=5.8 K 2024-11-14T04:35:24,114 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 41814b12fcfaad453a8c9184c10c8a26 in 36ms, sequenceid=32, compaction requested=false; wal=null 2024-11-14T04:35:24,115 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/0000000000000000032 2024-11-14T04:35:24,117 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,117 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,117 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:24,119 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41814b12fcfaad453a8c9184c10c8a26 2024-11-14T04:35:24,121 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/41814b12fcfaad453a8c9184c10c8a26/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-14T04:35:24,122 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41814b12fcfaad453a8c9184c10c8a26; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66691513, jitterRate=-0.006219014525413513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:24,123 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41814b12fcfaad453a8c9184c10c8a26: Writing region info on filesystem at 1731558924021Initializing all the Stores at 1731558924022 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558924022Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558924022Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558924022Obtaining lock to block concurrent updates at 1731558924078 (+56 ms)Preparing flush snapshotting stores in 41814b12fcfaad453a8c9184c10c8a26 at 1731558924078Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731558924078Flushing stores of testReplayEditsWrittenViaHRegion,,1731558923624.41814b12fcfaad453a8c9184c10c8a26. at 1731558924078Flushing 41814b12fcfaad453a8c9184c10c8a26/b: creating writer at 1731558924078Flushing 41814b12fcfaad453a8c9184c10c8a26/b: appending metadata at 1731558924095 (+17 ms)Flushing 41814b12fcfaad453a8c9184c10c8a26/b: closing flushed file at 1731558924095Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26255dd4: reopening flushed file at 1731558924107 (+12 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 41814b12fcfaad453a8c9184c10c8a26 in 36ms, sequenceid=32, compaction requested=false; wal=null at 1731558924114 (+7 ms)Cleaning up temporary data from old regions at 1731558924117 (+3 ms)Region opened successfully at 1731558924122 (+5 ms) 2024-11-14T04:35:24,142 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=414 (was 405) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:41630 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:38966 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:48008 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:42578 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:59404 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:49974 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1119 (was 1043) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=6700 (was 6709) 2024-11-14T04:35:24,142 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1119 is superior to 1024 2024-11-14T04:35:24,153 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=414, OpenFileDescriptor=1119, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=6700 2024-11-14T04:35:24,153 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1119 is superior to 1024 2024-11-14T04:35:24,167 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:24,169 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:24,170 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:24,173 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-62217383, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/hregion-62217383, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:24,185 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-62217383/hregion-62217383.1731558924173, exclude list is [], retry=0 2024-11-14T04:35:24,188 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:24,188 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:24,189 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:24,215 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-62217383/hregion-62217383.1731558924173 2024-11-14T04:35:24,216 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855)] 2024-11-14T04:35:24,216 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b7d5bc86eaafaddf5ca89004f59d8cca, NAME => 'testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/hbase 2024-11-14T04:35:24,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741902_1080 (size=68) 2024-11-14T04:35:24,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741902_1080 (size=68) 2024-11-14T04:35:24,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741902_1080 (size=68) 2024-11-14T04:35:24,226 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:24,227 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,229 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5bc86eaafaddf5ca89004f59d8cca columnFamilyName a 2024-11-14T04:35:24,229 DEBUG [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:24,229 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(327): Store=b7d5bc86eaafaddf5ca89004f59d8cca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:24,229 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,231 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5bc86eaafaddf5ca89004f59d8cca columnFamilyName b 2024-11-14T04:35:24,231 DEBUG [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:24,231 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(327): Store=b7d5bc86eaafaddf5ca89004f59d8cca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:24,231 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,232 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5bc86eaafaddf5ca89004f59d8cca columnFamilyName c 2024-11-14T04:35:24,233 DEBUG [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:24,233 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(327): Store=b7d5bc86eaafaddf5ca89004f59d8cca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:24,233 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,234 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,235 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,236 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,236 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,236 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:24,237 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,240 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:24,240 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b7d5bc86eaafaddf5ca89004f59d8cca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59907191, jitterRate=-0.10731329023838043}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:24,241 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b7d5bc86eaafaddf5ca89004f59d8cca: Writing region info on filesystem at 1731558924226Initializing all the Stores at 1731558924227 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558924227Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558924227Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558924227Cleaning up temporary data from old regions at 1731558924236 (+9 ms)Region opened successfully at 1731558924241 (+5 ms) 2024-11-14T04:35:24,241 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b7d5bc86eaafaddf5ca89004f59d8cca, disabling compactions & flushes 2024-11-14T04:35:24,241 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. 2024-11-14T04:35:24,241 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. 2024-11-14T04:35:24,241 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. after waiting 0 ms 2024-11-14T04:35:24,241 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. 2024-11-14T04:35:24,242 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. 2024-11-14T04:35:24,242 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b7d5bc86eaafaddf5ca89004f59d8cca: Waiting for close lock at 1731558924241Disabling compacts and flushes for region at 1731558924241Disabling writes for close at 1731558924241Writing region close event to WAL at 1731558924242 (+1 ms)Closed at 1731558924242 2024-11-14T04:35:24,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741901_1079 (size=175) 2024-11-14T04:35:24,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741901_1079 (size=175) 2024-11-14T04:35:24,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741901_1079 (size=175) 2024-11-14T04:35:24,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:24,247 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-62217383:(num 1731558924173) 2024-11-14T04:35:24,247 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:24,249 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:24,262 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249, exclude list is [], retry=0 2024-11-14T04:35:24,265 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:24,265 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:24,266 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:24,293 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249 2024-11-14T04:35:24,294 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-14T04:35:24,362 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b7d5bc86eaafaddf5ca89004f59d8cca, NAME => 'testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:24,365 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,365 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:24,365 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,365 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,367 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,368 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5bc86eaafaddf5ca89004f59d8cca columnFamilyName a 2024-11-14T04:35:24,368 DEBUG [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:24,369 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(327): Store=b7d5bc86eaafaddf5ca89004f59d8cca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:24,369 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,370 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5bc86eaafaddf5ca89004f59d8cca columnFamilyName b 2024-11-14T04:35:24,370 DEBUG [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:24,371 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(327): Store=b7d5bc86eaafaddf5ca89004f59d8cca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:24,371 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,372 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5bc86eaafaddf5ca89004f59d8cca columnFamilyName c 2024-11-14T04:35:24,372 DEBUG [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:24,373 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(327): Store=b7d5bc86eaafaddf5ca89004f59d8cca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:24,373 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,374 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,375 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,376 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,376 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,377 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:24,378 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,379 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b7d5bc86eaafaddf5ca89004f59d8cca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71788428, jitterRate=0.06973093748092651}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:24,380 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:24,380 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b7d5bc86eaafaddf5ca89004f59d8cca: Running coprocessor pre-open hook at 1731558924365Writing region info on filesystem at 1731558924365Initializing all the Stores at 1731558924366 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558924366Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558924367 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558924367Cleaning up temporary data from old regions at 1731558924376 (+9 ms)Running coprocessor post-open hooks at 1731558924380 (+4 ms)Region opened successfully at 1731558924380 2024-11-14T04:35:24,394 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b7d5bc86eaafaddf5ca89004f59d8cca 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-14T04:35:24,395 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b7d5bc86eaafaddf5ca89004f59d8cca/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:25,396 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b7d5bc86eaafaddf5ca89004f59d8cca/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:25,658 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T04:35:26,398 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b7d5bc86eaafaddf5ca89004f59d8cca/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:27,400 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b7d5bc86eaafaddf5ca89004f59d8cca/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:28,401 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b7d5bc86eaafaddf5ca89004f59d8cca/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:29,403 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b7d5bc86eaafaddf5ca89004f59d8cca/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:30,404 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b7d5bc86eaafaddf5ca89004f59d8cca/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:31,405 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b7d5bc86eaafaddf5ca89004f59d8cca/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:32,407 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b7d5bc86eaafaddf5ca89004f59d8cca/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:32,608 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T04:35:33,408 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for b7d5bc86eaafaddf5ca89004f59d8cca/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:33,411 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b7d5bc86eaafaddf5ca89004f59d8cca: 2024-11-14T04:35:33,412 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. 2024-11-14T04:35:33,430 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b7d5bc86eaafaddf5ca89004f59d8cca: 2024-11-14T04:35:33,430 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-14T04:35:33,430 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b7d5bc86eaafaddf5ca89004f59d8cca, disabling compactions & flushes 2024-11-14T04:35:33,430 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. 2024-11-14T04:35:33,430 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. 2024-11-14T04:35:33,430 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. after waiting 0 ms 2024-11-14T04:35:33,430 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. 2024-11-14T04:35:33,431 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. 2024-11-14T04:35:33,431 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. 2024-11-14T04:35:33,431 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b7d5bc86eaafaddf5ca89004f59d8cca: Waiting for close lock at 1731558933430Running coprocessor pre-close hooks at 1731558933430Disabling compacts and flushes for region at 1731558933430Disabling writes for close at 1731558933430Writing region close event to WAL at 1731558933431 (+1 ms)Running coprocessor post-close hooks at 1731558933431Closed at 1731558933431 2024-11-14T04:35:33,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741903_1081 (size=4236) 2024-11-14T04:35:33,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741903_1081 (size=4236) 2024-11-14T04:35:33,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741903_1081 (size=4236) 2024-11-14T04:35:33,448 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249, size=4.1 K (4236bytes) 2024-11-14T04:35:33,448 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249 2024-11-14T04:35:33,449 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249 after 1ms 2024-11-14T04:35:33,474 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:33,474 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249 took 26ms 2024-11-14T04:35:33,476 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249 so closing down 2024-11-14T04:35:33,477 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:33,477 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731558924249.temp 2024-11-14T04:35:33,502 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000004-wal.1731558924249.temp 2024-11-14T04:35:33,502 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:33,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741904_1082 (size=3245) 2024-11-14T04:35:33,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741904_1082 (size=3245) 2024-11-14T04:35:33,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741904_1082 (size=3245) 2024-11-14T04:35:33,511 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000004-wal.1731558924249.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:33,512 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000004-wal.1731558924249.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000026 2024-11-14T04:35:33,512 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 38 ms; skipped=3; WAL=hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-14T04:35:33,512 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249, journal: Splitting hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249, size=4.1 K (4236bytes) at 1731558933448Finishing writing output for hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249 so closing down at 1731558933476 (+28 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000004-wal.1731558924249.temp at 1731558933502 (+26 ms)3 split writer threads finished at 1731558933502Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000004-wal.1731558924249.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731558933511 (+9 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000004-wal.1731558924249.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000026 at 1731558933512 (+1 ms)Processed 23 edits across 1 Regions in 38 ms; skipped=3; WAL=hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1731558933512 2024-11-14T04:35:33,514 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558924249 to hdfs://localhost:41731/hbase/oldWALs/wal.1731558924249 2024-11-14T04:35:33,515 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000026 2024-11-14T04:35:33,515 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:33,517 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:33,534 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558933518, exclude list is [], retry=0 2024-11-14T04:35:33,537 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:33,538 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:33,538 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:33,570 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558933518 2024-11-14T04:35:33,571 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:33,571 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b7d5bc86eaafaddf5ca89004f59d8cca, NAME => 'testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:33,571 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,572 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:33,572 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,572 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,573 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,575 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5bc86eaafaddf5ca89004f59d8cca columnFamilyName a 2024-11-14T04:35:33,575 DEBUG [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:33,575 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(327): Store=b7d5bc86eaafaddf5ca89004f59d8cca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:33,576 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,576 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5bc86eaafaddf5ca89004f59d8cca columnFamilyName b 2024-11-14T04:35:33,577 DEBUG [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:33,577 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(327): Store=b7d5bc86eaafaddf5ca89004f59d8cca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:33,577 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,578 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d5bc86eaafaddf5ca89004f59d8cca columnFamilyName c 2024-11-14T04:35:33,578 DEBUG [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:33,579 INFO [StoreOpener-b7d5bc86eaafaddf5ca89004f59d8cca-1 {}] regionserver.HStore(327): Store=b7d5bc86eaafaddf5ca89004f59d8cca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:33,579 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,580 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,582 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,582 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000026 2024-11-14T04:35:33,617 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:33,618 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000026 2024-11-14T04:35:33,619 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b7d5bc86eaafaddf5ca89004f59d8cca 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-14T04:35:33,640 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/.tmp/a/6cd7f335f77d40f2b2912db894d34cee is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731558933422/Put/seqid=0 2024-11-14T04:35:33,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741906_1084 (size=5523) 2024-11-14T04:35:33,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741906_1084 (size=5523) 2024-11-14T04:35:33,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741906_1084 (size=5523) 2024-11-14T04:35:33,646 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/.tmp/a/6cd7f335f77d40f2b2912db894d34cee 2024-11-14T04:35:33,671 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/.tmp/b/00d1cf55c1ab4616805dd7fba0a628ba is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731558933413/Put/seqid=0 2024-11-14T04:35:33,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741907_1085 (size=5524) 2024-11-14T04:35:33,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741907_1085 (size=5524) 2024-11-14T04:35:33,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741907_1085 (size=5524) 2024-11-14T04:35:33,679 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/.tmp/b/00d1cf55c1ab4616805dd7fba0a628ba 2024-11-14T04:35:33,702 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/.tmp/c/423d127e67d5402b956761d1dfee66cb is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731558933420/Put/seqid=0 2024-11-14T04:35:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741908_1086 (size=5457) 2024-11-14T04:35:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741908_1086 (size=5457) 2024-11-14T04:35:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741908_1086 (size=5457) 2024-11-14T04:35:33,710 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/.tmp/c/423d127e67d5402b956761d1dfee66cb 2024-11-14T04:35:33,716 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/.tmp/a/6cd7f335f77d40f2b2912db894d34cee as hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/a/6cd7f335f77d40f2b2912db894d34cee 2024-11-14T04:35:33,722 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/a/6cd7f335f77d40f2b2912db894d34cee, entries=7, sequenceid=26, filesize=5.4 K 2024-11-14T04:35:33,723 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/.tmp/b/00d1cf55c1ab4616805dd7fba0a628ba as hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/b/00d1cf55c1ab4616805dd7fba0a628ba 2024-11-14T04:35:33,729 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/b/00d1cf55c1ab4616805dd7fba0a628ba, entries=7, sequenceid=26, filesize=5.4 K 2024-11-14T04:35:33,730 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/.tmp/c/423d127e67d5402b956761d1dfee66cb as hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/c/423d127e67d5402b956761d1dfee66cb 2024-11-14T04:35:33,735 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/c/423d127e67d5402b956761d1dfee66cb, entries=6, sequenceid=26, filesize=5.3 K 2024-11-14T04:35:33,735 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for b7d5bc86eaafaddf5ca89004f59d8cca in 116ms, sequenceid=26, compaction requested=false; wal=null 2024-11-14T04:35:33,736 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/0000000000000000026 2024-11-14T04:35:33,737 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,737 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,738 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:33,739 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,741 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsAfterAbortingFlush/b7d5bc86eaafaddf5ca89004f59d8cca/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-14T04:35:33,742 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b7d5bc86eaafaddf5ca89004f59d8cca; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72803848, jitterRate=0.0848618745803833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:33,742 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7d5bc86eaafaddf5ca89004f59d8cca 2024-11-14T04:35:33,742 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b7d5bc86eaafaddf5ca89004f59d8cca: Running coprocessor pre-open hook at 1731558933572Writing region info on filesystem at 1731558933572Initializing all the Stores at 1731558933573 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558933573Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558933573Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558933573Obtaining lock to block concurrent updates at 1731558933619 (+46 ms)Preparing flush snapshotting stores in b7d5bc86eaafaddf5ca89004f59d8cca at 1731558933619Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731558933619Flushing stores of testReplayEditsAfterAbortingFlush,,1731558924168.b7d5bc86eaafaddf5ca89004f59d8cca. at 1731558933619Flushing b7d5bc86eaafaddf5ca89004f59d8cca/a: creating writer at 1731558933619Flushing b7d5bc86eaafaddf5ca89004f59d8cca/a: appending metadata at 1731558933639 (+20 ms)Flushing b7d5bc86eaafaddf5ca89004f59d8cca/a: closing flushed file at 1731558933639Flushing b7d5bc86eaafaddf5ca89004f59d8cca/b: creating writer at 1731558933652 (+13 ms)Flushing b7d5bc86eaafaddf5ca89004f59d8cca/b: appending metadata at 1731558933671 (+19 ms)Flushing b7d5bc86eaafaddf5ca89004f59d8cca/b: closing flushed file at 1731558933671Flushing b7d5bc86eaafaddf5ca89004f59d8cca/c: creating writer at 1731558933684 (+13 ms)Flushing b7d5bc86eaafaddf5ca89004f59d8cca/c: appending metadata at 1731558933701 (+17 ms)Flushing b7d5bc86eaafaddf5ca89004f59d8cca/c: closing flushed file at 1731558933701Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@474aef07: reopening flushed file at 1731558933716 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1393a84e: reopening flushed file at 1731558933722 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78dc8b9: reopening flushed file at 1731558933729 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for b7d5bc86eaafaddf5ca89004f59d8cca in 116ms, sequenceid=26, compaction requested=false; wal=null at 1731558933735 (+6 ms)Cleaning up temporary data from old regions at 1731558933737 (+2 ms)Running coprocessor post-open hooks at 1731558933742 (+5 ms)Region opened successfully at 1731558933742 2024-11-14T04:35:33,759 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=413 (was 414), OpenFileDescriptor=1178 (was 1119) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=318 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=6682 (was 6700) 2024-11-14T04:35:33,759 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-11-14T04:35:33,771 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=413, OpenFileDescriptor=1178, MaxFileDescriptor=1048576, SystemLoadAverage=318, ProcessCount=11, AvailableMemoryMB=6682 2024-11-14T04:35:33,771 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-11-14T04:35:33,784 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:33,786 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:33,786 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:33,789 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-27635532, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/hregion-27635532, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:33,801 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-27635532/hregion-27635532.1731558933789, exclude list is [], retry=0 2024-11-14T04:35:33,803 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:33,803 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:33,803 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:33,828 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-27635532/hregion-27635532.1731558933789 2024-11-14T04:35:33,828 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855)] 2024-11-14T04:35:33,828 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9913b7b9fc97f681429a66226722c3d2, NAME => 'testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/hbase 2024-11-14T04:35:33,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741910_1088 (size=61) 2024-11-14T04:35:33,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741910_1088 (size=61) 2024-11-14T04:35:33,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741910_1088 (size=61) 2024-11-14T04:35:33,839 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:33,840 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,841 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9913b7b9fc97f681429a66226722c3d2 columnFamilyName a 2024-11-14T04:35:33,841 DEBUG [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:33,842 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] regionserver.HStore(327): Store=9913b7b9fc97f681429a66226722c3d2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:33,842 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,842 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,843 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,843 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,843 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,844 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,846 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:33,846 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9913b7b9fc97f681429a66226722c3d2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66561501, jitterRate=-0.008156344294548035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:35:33,847 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9913b7b9fc97f681429a66226722c3d2: Writing region info on filesystem at 1731558933839Initializing all the Stores at 1731558933840 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558933840Cleaning up temporary data from old regions at 1731558933843 (+3 ms)Region opened successfully at 1731558933847 (+4 ms) 2024-11-14T04:35:33,847 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9913b7b9fc97f681429a66226722c3d2, disabling compactions & flushes 2024-11-14T04:35:33,847 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. 2024-11-14T04:35:33,847 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. 2024-11-14T04:35:33,847 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. after waiting 0 ms 2024-11-14T04:35:33,847 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. 2024-11-14T04:35:33,848 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. 2024-11-14T04:35:33,848 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9913b7b9fc97f681429a66226722c3d2: Waiting for close lock at 1731558933847Disabling compacts and flushes for region at 1731558933847Disabling writes for close at 1731558933847Writing region close event to WAL at 1731558933848 (+1 ms)Closed at 1731558933848 2024-11-14T04:35:33,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741909_1087 (size=175) 2024-11-14T04:35:33,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741909_1087 (size=175) 2024-11-14T04:35:33,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741909_1087 (size=175) 2024-11-14T04:35:33,854 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:33,854 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-27635532:(num 1731558933789) 2024-11-14T04:35:33,854 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:33,856 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:33,870 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856, exclude list is [], retry=0 2024-11-14T04:35:33,872 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:33,873 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:33,873 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:33,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-14T04:35:33,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-14T04:35:33,901 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856 2024-11-14T04:35:33,901 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-14T04:35:33,901 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9913b7b9fc97f681429a66226722c3d2, NAME => 'testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:33,901 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:33,902 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,902 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,903 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,904 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9913b7b9fc97f681429a66226722c3d2 columnFamilyName a 2024-11-14T04:35:33,904 DEBUG [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:33,905 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] regionserver.HStore(327): Store=9913b7b9fc97f681429a66226722c3d2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:33,905 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,905 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,906 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,907 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,907 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,909 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:33,909 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9913b7b9fc97f681429a66226722c3d2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74905072, jitterRate=0.11617255210876465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:35:33,910 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9913b7b9fc97f681429a66226722c3d2: Writing region info on filesystem at 1731558933902Initializing all the Stores at 1731558933903 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558933903Cleaning up temporary data from old regions at 1731558933907 (+4 ms)Region opened successfully at 1731558933910 (+3 ms) 2024-11-14T04:35:33,919 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9913b7b9fc97f681429a66226722c3d2, disabling compactions & flushes 2024-11-14T04:35:33,919 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. 2024-11-14T04:35:33,919 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. 2024-11-14T04:35:33,919 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. after waiting 0 ms 2024-11-14T04:35:33,919 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. 2024-11-14T04:35:33,919 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. 2024-11-14T04:35:33,920 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. 2024-11-14T04:35:33,920 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9913b7b9fc97f681429a66226722c3d2: Waiting for close lock at 1731558933919Disabling compacts and flushes for region at 1731558933919Disabling writes for close at 1731558933919Writing region close event to WAL at 1731558933919Closed at 1731558933919 2024-11-14T04:35:33,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741911_1089 (size=1795) 2024-11-14T04:35:33,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741911_1089 (size=1795) 2024-11-14T04:35:33,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741911_1089 (size=1795) 2024-11-14T04:35:33,944 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856, size=1.8 K (1795bytes) 2024-11-14T04:35:33,944 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856 2024-11-14T04:35:33,944 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856 after 0ms 2024-11-14T04:35:33,969 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:33,969 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856 took 25ms 2024-11-14T04:35:33,971 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856 so closing down 2024-11-14T04:35:33,971 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:33,972 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731558933856.temp 2024-11-14T04:35:33,998 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000003-wal.1731558933856.temp 2024-11-14T04:35:33,999 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:34,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741912_1090 (size=1795) 2024-11-14T04:35:34,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741912_1090 (size=1795) 2024-11-14T04:35:34,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741912_1090 (size=1795) 2024-11-14T04:35:34,005 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000003-wal.1731558933856.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:34,006 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000003-wal.1731558933856.temp to hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000012 2024-11-14T04:35:34,006 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-14T04:35:34,006 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856, journal: Splitting hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856, size=1.8 K (1795bytes) at 1731558933944Finishing writing output for hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856 so closing down at 1731558933971 (+27 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000003-wal.1731558933856.temp at 1731558933998 (+27 ms)3 split writer threads finished at 1731558933999 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000003-wal.1731558933856.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731558934005 (+6 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000003-wal.1731558933856.temp to hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000012 at 1731558934006 (+1 ms)Processed 10 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1731558934006 2024-11-14T04:35:34,008 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558933856 to hdfs://localhost:41731/hbase/oldWALs/wal.1731558933856 2024-11-14T04:35:34,009 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000012 2024-11-14T04:35:34,035 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:34,386 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:34,388 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:34,400 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558934388, exclude list is [], retry=0 2024-11-14T04:35:34,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:34,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:34,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:34,426 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558934388 2024-11-14T04:35:34,426 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105)] 2024-11-14T04:35:34,426 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9913b7b9fc97f681429a66226722c3d2, NAME => 'testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:34,426 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:34,426 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,426 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,429 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,431 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9913b7b9fc97f681429a66226722c3d2 columnFamilyName a 2024-11-14T04:35:34,431 DEBUG [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:34,432 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] regionserver.HStore(327): Store=9913b7b9fc97f681429a66226722c3d2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:34,432 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,433 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,434 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,435 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000012 2024-11-14T04:35:34,462 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:34,463 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000012 2024-11-14T04:35:34,464 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9913b7b9fc97f681429a66226722c3d2 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-14T04:35:34,481 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/.tmp/a/deebbc59201e4078b2d2ff093821d945 is 79, key is testDatalossWhenInputError/a:x0/1731558933910/Put/seqid=0 2024-11-14T04:35:34,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741914_1092 (size=5808) 2024-11-14T04:35:34,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741914_1092 (size=5808) 2024-11-14T04:35:34,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741914_1092 (size=5808) 2024-11-14T04:35:34,521 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/.tmp/a/deebbc59201e4078b2d2ff093821d945 2024-11-14T04:35:34,532 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/.tmp/a/deebbc59201e4078b2d2ff093821d945 as hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/a/deebbc59201e4078b2d2ff093821d945 2024-11-14T04:35:34,542 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/a/deebbc59201e4078b2d2ff093821d945, entries=10, sequenceid=12, filesize=5.7 K 2024-11-14T04:35:34,542 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9913b7b9fc97f681429a66226722c3d2 in 79ms, sequenceid=12, compaction requested=false; wal=null 2024-11-14T04:35:34,543 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/0000000000000000012 2024-11-14T04:35:34,544 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,544 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,547 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,550 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-14T04:35:34,551 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9913b7b9fc97f681429a66226722c3d2; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70987994, jitterRate=0.05780354142189026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:35:34,552 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9913b7b9fc97f681429a66226722c3d2: Writing region info on filesystem at 1731558934426Initializing all the Stores at 1731558934429 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558934429Obtaining lock to block concurrent updates at 1731558934464 (+35 ms)Preparing flush snapshotting stores in 9913b7b9fc97f681429a66226722c3d2 at 1731558934464Finished memstore snapshotting testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731558934464Flushing stores of testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2. at 1731558934464Flushing 9913b7b9fc97f681429a66226722c3d2/a: creating writer at 1731558934464Flushing 9913b7b9fc97f681429a66226722c3d2/a: appending metadata at 1731558934480 (+16 ms)Flushing 9913b7b9fc97f681429a66226722c3d2/a: closing flushed file at 1731558934480Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@601079c5: reopening flushed file at 1731558934530 (+50 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9913b7b9fc97f681429a66226722c3d2 in 79ms, sequenceid=12, compaction requested=false; wal=null at 1731558934542 (+12 ms)Cleaning up temporary data from old regions at 1731558934544 (+2 ms)Region opened successfully at 1731558934551 (+7 ms) 2024-11-14T04:35:34,555 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9913b7b9fc97f681429a66226722c3d2, NAME => 'testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:34,555 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731558933785.9913b7b9fc97f681429a66226722c3d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:34,555 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,555 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,557 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,558 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9913b7b9fc97f681429a66226722c3d2 columnFamilyName a 2024-11-14T04:35:34,558 DEBUG [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:34,563 DEBUG [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/a/deebbc59201e4078b2d2ff093821d945 2024-11-14T04:35:34,563 INFO [StoreOpener-9913b7b9fc97f681429a66226722c3d2-1 {}] regionserver.HStore(327): Store=9913b7b9fc97f681429a66226722c3d2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:34,563 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,564 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,566 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,566 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,566 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,568 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9913b7b9fc97f681429a66226722c3d2 2024-11-14T04:35:34,570 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testDatalossWhenInputError/9913b7b9fc97f681429a66226722c3d2/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-14T04:35:34,571 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9913b7b9fc97f681429a66226722c3d2; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75082742, jitterRate=0.11882004141807556}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T04:35:34,572 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9913b7b9fc97f681429a66226722c3d2: Writing region info on filesystem at 1731558934556Initializing all the Stores at 1731558934557 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558934557Cleaning up temporary data from old regions at 1731558934566 (+9 ms)Region opened successfully at 1731558934572 (+6 ms) 2024-11-14T04:35:34,589 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=423 (was 413) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:59454 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:49344 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:39684 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-954332279-172.17.0.3-1731558898852:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:39750 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:59372 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:49398 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1260 (was 1178) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=318 (was 318), ProcessCount=11 (was 11), AvailableMemoryMB=6659 (was 6682) 2024-11-14T04:35:34,589 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-11-14T04:35:34,604 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=423, OpenFileDescriptor=1260, MaxFileDescriptor=1048576, SystemLoadAverage=318, ProcessCount=11, AvailableMemoryMB=6658 2024-11-14T04:35:34,604 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-11-14T04:35:34,623 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:34,625 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:34,626 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:34,629 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19589810, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/hregion-19589810, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:34,645 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19589810/hregion-19589810.1731558934629, exclude list is [], retry=0 2024-11-14T04:35:34,647 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:34,647 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:34,647 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:34,673 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19589810/hregion-19589810.1731558934629 2024-11-14T04:35:34,673 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:34,673 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => fc810b9d6433858b78b51b4124ae3a19, NAME => 'testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/hbase 2024-11-14T04:35:34,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741916_1094 (size=63) 2024-11-14T04:35:34,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741916_1094 (size=63) 2024-11-14T04:35:34,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741916_1094 (size=63) 2024-11-14T04:35:34,682 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:34,684 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,685 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc810b9d6433858b78b51b4124ae3a19 columnFamilyName a 2024-11-14T04:35:34,685 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:34,686 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(327): Store=fc810b9d6433858b78b51b4124ae3a19/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:34,686 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,687 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc810b9d6433858b78b51b4124ae3a19 columnFamilyName b 2024-11-14T04:35:34,687 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:34,688 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(327): Store=fc810b9d6433858b78b51b4124ae3a19/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:34,688 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,689 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc810b9d6433858b78b51b4124ae3a19 columnFamilyName c 2024-11-14T04:35:34,689 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:34,690 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(327): Store=fc810b9d6433858b78b51b4124ae3a19/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:34,690 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,691 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,691 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,692 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,692 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,693 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:34,694 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,697 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:34,697 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fc810b9d6433858b78b51b4124ae3a19; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75074527, jitterRate=0.1186976283788681}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:34,698 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fc810b9d6433858b78b51b4124ae3a19: Writing region info on filesystem at 1731558934683Initializing all the Stores at 1731558934683Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558934683Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558934684 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558934684Cleaning up temporary data from old regions at 1731558934692 (+8 ms)Region opened successfully at 1731558934698 (+6 ms) 2024-11-14T04:35:34,698 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fc810b9d6433858b78b51b4124ae3a19, disabling compactions & flushes 2024-11-14T04:35:34,698 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. 2024-11-14T04:35:34,698 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. 2024-11-14T04:35:34,698 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. after waiting 0 ms 2024-11-14T04:35:34,698 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. 2024-11-14T04:35:34,699 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. 2024-11-14T04:35:34,699 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fc810b9d6433858b78b51b4124ae3a19: Waiting for close lock at 1731558934698Disabling compacts and flushes for region at 1731558934698Disabling writes for close at 1731558934698Writing region close event to WAL at 1731558934699 (+1 ms)Closed at 1731558934699 2024-11-14T04:35:34,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741915_1093 (size=175) 2024-11-14T04:35:34,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741915_1093 (size=175) 2024-11-14T04:35:34,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741915_1093 (size=175) 2024-11-14T04:35:34,704 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:34,704 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19589810:(num 1731558934629) 2024-11-14T04:35:34,704 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:34,706 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:34,723 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706, exclude list is [], retry=0 2024-11-14T04:35:34,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:34,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:34,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:34,756 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 2024-11-14T04:35:34,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:34,757 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fc810b9d6433858b78b51b4124ae3a19, NAME => 'testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:34,757 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:34,757 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,757 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,758 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,759 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc810b9d6433858b78b51b4124ae3a19 columnFamilyName a 2024-11-14T04:35:34,759 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:34,760 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(327): Store=fc810b9d6433858b78b51b4124ae3a19/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:34,760 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,761 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc810b9d6433858b78b51b4124ae3a19 columnFamilyName b 2024-11-14T04:35:34,761 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:34,761 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(327): Store=fc810b9d6433858b78b51b4124ae3a19/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:34,762 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,762 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc810b9d6433858b78b51b4124ae3a19 columnFamilyName c 2024-11-14T04:35:34,762 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:34,763 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(327): Store=fc810b9d6433858b78b51b4124ae3a19/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:34,763 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,763 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,765 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,766 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,766 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,766 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:34,767 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:34,768 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fc810b9d6433858b78b51b4124ae3a19; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74838025, jitterRate=0.11517347395420074}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:34,769 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fc810b9d6433858b78b51b4124ae3a19: Writing region info on filesystem at 1731558934757Initializing all the Stores at 1731558934758 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558934758Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558934758Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558934758Cleaning up temporary data from old regions at 1731558934766 (+8 ms)Region opened successfully at 1731558934769 (+3 ms) 2024-11-14T04:35:34,772 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731558934772/Put/seqid=0 2024-11-14T04:35:34,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741918_1096 (size=4875) 2024-11-14T04:35:34,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741918_1096 (size=4875) 2024-11-14T04:35:34,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741918_1096 (size=4875) 2024-11-14T04:35:34,784 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731558934784/Put/seqid=0 2024-11-14T04:35:34,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741919_1097 (size=4875) 2024-11-14T04:35:34,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741919_1097 (size=4875) 2024-11-14T04:35:34,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741919_1097 (size=4875) 2024-11-14T04:35:34,793 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731558934792/Put/seqid=0 2024-11-14T04:35:34,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741920_1098 (size=4875) 2024-11-14T04:35:34,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741920_1098 (size=4875) 2024-11-14T04:35:34,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741920_1098 (size=4875) 2024-11-14T04:35:34,799 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in fc810b9d6433858b78b51b4124ae3a19/a 2024-11-14T04:35:34,803 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-14T04:35:34,803 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T04:35:34,803 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in fc810b9d6433858b78b51b4124ae3a19/a 2024-11-14T04:35:34,807 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-14T04:35:34,807 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T04:35:34,807 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in fc810b9d6433858b78b51b4124ae3a19/a 2024-11-14T04:35:34,811 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-14T04:35:34,811 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T04:35:34,811 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fc810b9d6433858b78b51b4124ae3a19 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-14T04:35:34,826 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/.tmp/a/f6fb14884a5e44f8887da47074d75cbb is 55, key is testCompactedBulkLoadedFiles/a:a/1731558934769/Put/seqid=0 2024-11-14T04:35:34,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741921_1099 (size=5107) 2024-11-14T04:35:34,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741921_1099 (size=5107) 2024-11-14T04:35:34,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741921_1099 (size=5107) 2024-11-14T04:35:34,836 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/.tmp/a/f6fb14884a5e44f8887da47074d75cbb 2024-11-14T04:35:34,842 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/.tmp/a/f6fb14884a5e44f8887da47074d75cbb as hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/f6fb14884a5e44f8887da47074d75cbb 2024-11-14T04:35:34,848 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/f6fb14884a5e44f8887da47074d75cbb, entries=1, sequenceid=4, filesize=5.0 K 2024-11-14T04:35:34,849 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for fc810b9d6433858b78b51b4124ae3a19 in 38ms, sequenceid=4, compaction requested=false 2024-11-14T04:35:34,849 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fc810b9d6433858b78b51b4124ae3a19: 2024-11-14T04:35:34,851 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_ 2024-11-14T04:35:34,852 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/14fe0cbba04e47739a3a1c55a233efea_SeqId_4_ 2024-11-14T04:35:34,853 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_ 2024-11-14T04:35:34,854 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile0 into fc810b9d6433858b78b51b4124ae3a19/a as hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_ - updating store file list. 2024-11-14T04:35:34,858 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T04:35:34,859 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_ into fc810b9d6433858b78b51b4124ae3a19/a 2024-11-14T04:35:34,859 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile0 into fc810b9d6433858b78b51b4124ae3a19/a (new location: hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_) 2024-11-14T04:35:34,860 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile1 into fc810b9d6433858b78b51b4124ae3a19/a as hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/14fe0cbba04e47739a3a1c55a233efea_SeqId_4_ - updating store file list. 2024-11-14T04:35:34,864 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 14fe0cbba04e47739a3a1c55a233efea_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T04:35:34,865 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/14fe0cbba04e47739a3a1c55a233efea_SeqId_4_ into fc810b9d6433858b78b51b4124ae3a19/a 2024-11-14T04:35:34,865 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile1 into fc810b9d6433858b78b51b4124ae3a19/a (new location: hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/14fe0cbba04e47739a3a1c55a233efea_SeqId_4_) 2024-11-14T04:35:34,866 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile2 into fc810b9d6433858b78b51b4124ae3a19/a as hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_ - updating store file list. 2024-11-14T04:35:34,870 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T04:35:34,870 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_ into fc810b9d6433858b78b51b4124ae3a19/a 2024-11-14T04:35:34,870 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41731/hbase/testCompactedBulkLoadedFiles/hfile2 into fc810b9d6433858b78b51b4124ae3a19/a (new location: hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_) 2024-11-14T04:35:34,877 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T04:35:34,877 DEBUG [Time-limited test {}] regionserver.HStore(1541): fc810b9d6433858b78b51b4124ae3a19/a is initiating major compaction (all files) 2024-11-14T04:35:34,877 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of fc810b9d6433858b78b51b4124ae3a19/a in testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. 2024-11-14T04:35:34,877 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/f6fb14884a5e44f8887da47074d75cbb, hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_, hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/14fe0cbba04e47739a3a1c55a233efea_SeqId_4_, hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_] into tmpdir=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/.tmp, totalSize=19.3 K 2024-11-14T04:35:34,878 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f6fb14884a5e44f8887da47074d75cbb, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731558934769 2024-11-14T04:35:34,879 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T04:35:34,879 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 14fe0cbba04e47739a3a1c55a233efea_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T04:35:34,880 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T04:35:34,893 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/.tmp/a/a3e43db6221c4228a0e74e0b196d80ec is 55, key is testCompactedBulkLoadedFiles/a:a/1731558934769/Put/seqid=0 2024-11-14T04:35:34,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741922_1100 (size=6154) 2024-11-14T04:35:34,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741922_1100 (size=6154) 2024-11-14T04:35:34,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741922_1100 (size=6154) 2024-11-14T04:35:34,906 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/.tmp/a/a3e43db6221c4228a0e74e0b196d80ec as hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/a3e43db6221c4228a0e74e0b196d80ec 2024-11-14T04:35:34,912 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in fc810b9d6433858b78b51b4124ae3a19/a of fc810b9d6433858b78b51b4124ae3a19 into a3e43db6221c4228a0e74e0b196d80ec(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T04:35:34,912 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for fc810b9d6433858b78b51b4124ae3a19: 2024-11-14T04:35:34,912 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-14T04:35:34,912 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-14T04:35:34,943 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706, size=0 (0bytes) 2024-11-14T04:35:34,943 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 might be still open, length is 0 2024-11-14T04:35:34,943 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 2024-11-14T04:35:34,944 WARN [IPC Server handler 4 on default port 41731 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-14T04:35:34,944 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 after 1ms 2024-11-14T04:35:35,559 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:59500 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59500 dst: /127.0.0.1:33533 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33533 remote=/127.0.0.1:59500]. Total timeout mills is 60000, 59354 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:35,559 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:49446 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37377:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49446 dst: /127.0.0.1:37377 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:35,559 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:39780 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39780 dst: /127.0.0.1:36191 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:35,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741917_1101 (size=1794) 2024-11-14T04:35:35,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741917_1101 (size=1794) 2024-11-14T04:35:38,945 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 after 4002ms 2024-11-14T04:35:38,964 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T04:35:38,983 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:38,983 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 took 4040ms 2024-11-14T04:35:38,985 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706; continuing. 2024-11-14T04:35:38,986 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 so closing down 2024-11-14T04:35:38,986 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:38,987 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731558934706.temp 2024-11-14T04:35:39,012 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000003-wal.1731558934706.temp 2024-11-14T04:35:39,012 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:39,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741923_1102 (size=800) 2024-11-14T04:35:39,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741923_1102 (size=800) 2024-11-14T04:35:39,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741923_1102 (size=800) 2024-11-14T04:35:39,018 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000003-wal.1731558934706.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:39,019 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000003-wal.1731558934706.temp to hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000008 2024-11-14T04:35:39,019 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 35 ms; skipped=3; WAL=hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T04:35:39,019 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706, journal: Splitting hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706, size=0 (0bytes) at 1731558934943Finishing writing output for hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 so closing down at 1731558938986 (+4043 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000003-wal.1731558934706.temp at 1731558939012 (+26 ms)3 split writer threads finished at 1731558939012Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000003-wal.1731558934706.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731558939018 (+6 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000003-wal.1731558934706.temp to hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000008 at 1731558939019 (+1 ms)Processed 5 edits across 1 Regions in 35 ms; skipped=3; WAL=hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706, size=0, length=0, corrupted=false, cancelled=false at 1731558939019 2024-11-14T04:35:39,021 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 to hdfs://localhost:41731/hbase/oldWALs/wal.1731558934706 2024-11-14T04:35:39,022 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000008 2024-11-14T04:35:39,022 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:39,024 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:39,037 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558939024, exclude list is [], retry=0 2024-11-14T04:35:39,039 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:39,039 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:39,039 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:39,065 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558939024 2024-11-14T04:35:39,065 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855)] 2024-11-14T04:35:39,065 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => fc810b9d6433858b78b51b4124ae3a19, NAME => 'testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:39,066 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:39,066 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,066 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,067 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,068 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc810b9d6433858b78b51b4124ae3a19 columnFamilyName a 2024-11-14T04:35:39,068 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,074 DEBUG [StoreFileOpener-fc810b9d6433858b78b51b4124ae3a19-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T04:35:39,074 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_ 2024-11-14T04:35:39,078 DEBUG [StoreFileOpener-fc810b9d6433858b78b51b4124ae3a19-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 14fe0cbba04e47739a3a1c55a233efea_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T04:35:39,078 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/14fe0cbba04e47739a3a1c55a233efea_SeqId_4_ 2024-11-14T04:35:39,082 DEBUG [StoreFileOpener-fc810b9d6433858b78b51b4124ae3a19-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T04:35:39,082 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_ 2024-11-14T04:35:39,087 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/a3e43db6221c4228a0e74e0b196d80ec 2024-11-14T04:35:39,090 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/f6fb14884a5e44f8887da47074d75cbb 2024-11-14T04:35:39,091 WARN [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3528dff8 2024-11-14T04:35:39,091 WARN [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/14fe0cbba04e47739a3a1c55a233efea_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3528dff8 2024-11-14T04:35:39,091 WARN [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3528dff8 2024-11-14T04:35:39,091 WARN [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/f6fb14884a5e44f8887da47074d75cbb from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3528dff8 2024-11-14T04:35:39,091 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_, hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/14fe0cbba04e47739a3a1c55a233efea_SeqId_4_, hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_, hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/f6fb14884a5e44f8887da47074d75cbb] to archive 2024-11-14T04:35:39,092 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T04:35:39,094 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_ to hdfs://localhost:41731/hbase/archive/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_ 2024-11-14T04:35:39,095 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/14fe0cbba04e47739a3a1c55a233efea_SeqId_4_ to hdfs://localhost:41731/hbase/archive/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/14fe0cbba04e47739a3a1c55a233efea_SeqId_4_ 2024-11-14T04:35:39,096 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_ to hdfs://localhost:41731/hbase/archive/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_ 2024-11-14T04:35:39,097 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/f6fb14884a5e44f8887da47074d75cbb to hdfs://localhost:41731/hbase/archive/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/a/f6fb14884a5e44f8887da47074d75cbb 2024-11-14T04:35:39,097 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(327): Store=fc810b9d6433858b78b51b4124ae3a19/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,097 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,098 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc810b9d6433858b78b51b4124ae3a19 columnFamilyName b 2024-11-14T04:35:39,098 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,099 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(327): Store=fc810b9d6433858b78b51b4124ae3a19/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,099 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,099 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc810b9d6433858b78b51b4124ae3a19 columnFamilyName c 2024-11-14T04:35:39,099 DEBUG [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,100 INFO [StoreOpener-fc810b9d6433858b78b51b4124ae3a19-1 {}] regionserver.HStore(327): Store=fc810b9d6433858b78b51b4124ae3a19/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,100 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,100 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,102 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,102 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000008 2024-11-14T04:35:39,126 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:39,127 DEBUG [Time-limited test {}] regionserver.HRegion(5836): fc810b9d6433858b78b51b4124ae3a19 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "fc810b9d6433858b78b51b4124ae3a19" family_name: "a" compaction_input: "f6fb14884a5e44f8887da47074d75cbb" compaction_input: "2e30d86521a64cb99c16eaf27215e8cf_SeqId_4_" compaction_input: "14fe0cbba04e47739a3a1c55a233efea_SeqId_4_" compaction_input: "0b7a3a3c5b9044d4bb8989f6dc6d44cc_SeqId_4_" compaction_output: "a3e43db6221c4228a0e74e0b196d80ec" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-14T04:35:39,127 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-14T04:35:39,127 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000008 2024-11-14T04:35:39,128 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/0000000000000000008 2024-11-14T04:35:39,129 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,129 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,130 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:39,132 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for fc810b9d6433858b78b51b4124ae3a19 2024-11-14T04:35:39,134 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testCompactedBulkLoadedFiles/fc810b9d6433858b78b51b4124ae3a19/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-14T04:35:39,135 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened fc810b9d6433858b78b51b4124ae3a19; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74355379, jitterRate=0.10798148810863495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:39,135 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for fc810b9d6433858b78b51b4124ae3a19: Writing region info on filesystem at 1731558939066Initializing all the Stores at 1731558939067 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939067Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939067Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939067Cleaning up temporary data from old regions at 1731558939129 (+62 ms)Region opened successfully at 1731558939135 (+6 ms) 2024-11-14T04:35:39,137 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing fc810b9d6433858b78b51b4124ae3a19, disabling compactions & flushes 2024-11-14T04:35:39,137 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. 2024-11-14T04:35:39,137 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. 2024-11-14T04:35:39,138 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. after waiting 0 ms 2024-11-14T04:35:39,138 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. 2024-11-14T04:35:39,138 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731558934624.fc810b9d6433858b78b51b4124ae3a19. 2024-11-14T04:35:39,138 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for fc810b9d6433858b78b51b4124ae3a19: Waiting for close lock at 1731558939137Disabling compacts and flushes for region at 1731558939137Disabling writes for close at 1731558939138 (+1 ms)Writing region close event to WAL at 1731558939138Closed at 1731558939138 2024-11-14T04:35:39,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741924_1103 (size=175) 2024-11-14T04:35:39,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741924_1103 (size=175) 2024-11-14T04:35:39,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741924_1103 (size=175) 2024-11-14T04:35:39,143 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:39,143 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731558939024) 2024-11-14T04:35:39,154 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=438 (was 423) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1433296787_22 at /127.0.0.1:59568 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:41731 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:41731 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1433296787_22 at /127.0.0.1:49512 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1433296787_22 at /127.0.0.1:39834 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1260) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=300 (was 318), ProcessCount=11 (was 11), AvailableMemoryMB=6640 (was 6658) 2024-11-14T04:35:39,154 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-14T04:35:39,165 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=300, ProcessCount=11, AvailableMemoryMB=6639 2024-11-14T04:35:39,165 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-11-14T04:35:39,178 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:39,179 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T04:35:39,180 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T04:35:39,182 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-03616481, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/hregion-03616481, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:39,197 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-03616481/hregion-03616481.1731558939183, exclude list is [], retry=0 2024-11-14T04:35:39,199 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:39,199 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:39,200 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:39,226 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-03616481/hregion-03616481.1731558939183 2024-11-14T04:35:39,226 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:39,227 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3ec537b454cc5cf85ff686857ee98810, NAME => 'testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41731/hbase 2024-11-14T04:35:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741926_1105 (size=67) 2024-11-14T04:35:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741926_1105 (size=67) 2024-11-14T04:35:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741926_1105 (size=67) 2024-11-14T04:35:39,235 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:39,236 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,237 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName a 2024-11-14T04:35:39,237 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,237 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,238 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,239 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName b 2024-11-14T04:35:39,239 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,240 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,240 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,241 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName c 2024-11-14T04:35:39,241 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,242 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,242 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,243 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,243 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,244 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,244 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,245 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:39,246 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,249 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T04:35:39,249 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3ec537b454cc5cf85ff686857ee98810; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60881323, jitterRate=-0.09279759228229523}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:39,249 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3ec537b454cc5cf85ff686857ee98810: Writing region info on filesystem at 1731558939235Initializing all the Stores at 1731558939235Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939235Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939236 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939236Cleaning up temporary data from old regions at 1731558939244 (+8 ms)Region opened successfully at 1731558939249 (+5 ms) 2024-11-14T04:35:39,250 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3ec537b454cc5cf85ff686857ee98810, disabling compactions & flushes 2024-11-14T04:35:39,250 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:39,250 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:39,250 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. after waiting 0 ms 2024-11-14T04:35:39,250 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:39,250 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:39,250 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3ec537b454cc5cf85ff686857ee98810: Waiting for close lock at 1731558939250Disabling compacts and flushes for region at 1731558939250Disabling writes for close at 1731558939250Writing region close event to WAL at 1731558939250Closed at 1731558939250 2024-11-14T04:35:39,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741925_1104 (size=175) 2024-11-14T04:35:39,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741925_1104 (size=175) 2024-11-14T04:35:39,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741925_1104 (size=175) 2024-11-14T04:35:39,259 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:39,259 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-03616481:(num 1731558939183) 2024-11-14T04:35:39,259 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:39,261 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:39,273 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262, exclude list is [], retry=0 2024-11-14T04:35:39,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:39,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:39,276 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:39,300 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262 2024-11-14T04:35:39,301 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:46855:46855)] 2024-11-14T04:35:39,301 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ec537b454cc5cf85ff686857ee98810, NAME => 'testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:39,301 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:39,301 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,301 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,302 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,303 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName a 2024-11-14T04:35:39,303 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,304 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,304 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,305 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName b 2024-11-14T04:35:39,305 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,305 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,305 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,306 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName c 2024-11-14T04:35:39,307 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,307 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,307 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,308 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,309 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,310 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,310 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,310 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:39,312 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,313 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3ec537b454cc5cf85ff686857ee98810; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66246533, jitterRate=-0.012849733233451843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:39,313 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3ec537b454cc5cf85ff686857ee98810: Writing region info on filesystem at 1731558939301Initializing all the Stores at 1731558939302 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939302Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939302Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939302Cleaning up temporary data from old regions at 1731558939310 (+8 ms)Region opened successfully at 1731558939313 (+3 ms) 2024-11-14T04:35:39,320 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3ec537b454cc5cf85ff686857ee98810 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-14T04:35:39,335 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/a/5edb967245bb4ef2bfa1bfc775a9e51b is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731558939313/Put/seqid=0 2024-11-14T04:35:39,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741928_1107 (size=5958) 2024-11-14T04:35:39,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741928_1107 (size=5958) 2024-11-14T04:35:39,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741928_1107 (size=5958) 2024-11-14T04:35:39,342 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/a/5edb967245bb4ef2bfa1bfc775a9e51b 2024-11-14T04:35:39,347 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/a/5edb967245bb4ef2bfa1bfc775a9e51b as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/a/5edb967245bb4ef2bfa1bfc775a9e51b 2024-11-14T04:35:39,352 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/a/5edb967245bb4ef2bfa1bfc775a9e51b, entries=10, sequenceid=13, filesize=5.8 K 2024-11-14T04:35:39,353 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 3ec537b454cc5cf85ff686857ee98810 in 33ms, sequenceid=13, compaction requested=false 2024-11-14T04:35:39,353 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3ec537b454cc5cf85ff686857ee98810: 2024-11-14T04:35:39,369 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3ec537b454cc5cf85ff686857ee98810, disabling compactions & flushes 2024-11-14T04:35:39,369 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:39,369 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:39,369 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. after waiting 0 ms 2024-11-14T04:35:39,369 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:39,370 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:39,370 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:39,370 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3ec537b454cc5cf85ff686857ee98810: Waiting for close lock at 1731558939369Disabling compacts and flushes for region at 1731558939369Disabling writes for close at 1731558939369Writing region close event to WAL at 1731558939370 (+1 ms)Closed at 1731558939370 2024-11-14T04:35:39,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741927_1106 (size=6243) 2024-11-14T04:35:39,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741927_1106 (size=6243) 2024-11-14T04:35:39,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741927_1106 (size=6243) 2024-11-14T04:35:39,392 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262, size=6.1 K (6243bytes) 2024-11-14T04:35:39,392 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262 2024-11-14T04:35:39,392 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262 after 0ms 2024-11-14T04:35:39,417 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:39,417 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262 took 26ms 2024-11-14T04:35:39,419 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262 so closing down 2024-11-14T04:35:39,419 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:39,420 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731558939262.temp 2024-11-14T04:35:39,444 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000003-wal.1731558939262.temp 2024-11-14T04:35:39,444 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:39,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741929_1108 (size=5575) 2024-11-14T04:35:39,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741929_1108 (size=5575) 2024-11-14T04:35:39,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741929_1108 (size=5575) 2024-11-14T04:35:39,451 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000003-wal.1731558939262.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:39,452 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000003-wal.1731558939262.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000035 2024-11-14T04:35:39,452 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-14T04:35:39,452 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262, journal: Splitting hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262, size=6.1 K (6243bytes) at 1731558939392Finishing writing output for hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262 so closing down at 1731558939419 (+27 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000003-wal.1731558939262.temp at 1731558939444 (+25 ms)3 split writer threads finished at 1731558939444Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000003-wal.1731558939262.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731558939451 (+7 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000003-wal.1731558939262.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000035 at 1731558939452 (+1 ms)Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1731558939452 2024-11-14T04:35:39,454 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939262 to hdfs://localhost:41731/hbase/oldWALs/wal.1731558939262 2024-11-14T04:35:39,455 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000035 2024-11-14T04:35:39,455 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:39,456 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:39,471 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457, exclude list is [], retry=0 2024-11-14T04:35:39,474 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:39,474 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:39,474 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:39,554 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 2024-11-14T04:35:39,555 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46855:46855),(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129)] 2024-11-14T04:35:39,555 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ec537b454cc5cf85ff686857ee98810, NAME => 'testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810.', STARTKEY => '', ENDKEY => ''} 2024-11-14T04:35:39,555 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:39,555 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,555 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,556 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,557 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName a 2024-11-14T04:35:39,557 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,562 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/a/5edb967245bb4ef2bfa1bfc775a9e51b 2024-11-14T04:35:39,562 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,562 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,563 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName b 2024-11-14T04:35:39,563 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,564 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,564 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,564 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName c 2024-11-14T04:35:39,564 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:39,565 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:39,565 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,565 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,567 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,567 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000035 2024-11-14T04:35:39,593 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:39,594 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000035 2024-11-14T04:35:39,594 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3ec537b454cc5cf85ff686857ee98810 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-14T04:35:39,607 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/b/c3130e7af5254ba2bda97dd102f52c9c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731558939353/Put/seqid=0 2024-11-14T04:35:39,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741931_1110 (size=5958) 2024-11-14T04:35:39,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741931_1110 (size=5958) 2024-11-14T04:35:39,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741931_1110 (size=5958) 2024-11-14T04:35:39,614 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/b/c3130e7af5254ba2bda97dd102f52c9c 2024-11-14T04:35:39,634 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/c/6b0152f6a6fb4e5287ce6567b0eca5f2 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731558939360/Put/seqid=0 2024-11-14T04:35:39,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741932_1111 (size=5958) 2024-11-14T04:35:39,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741932_1111 (size=5958) 2024-11-14T04:35:39,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741932_1111 (size=5958) 2024-11-14T04:35:39,640 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/c/6b0152f6a6fb4e5287ce6567b0eca5f2 2024-11-14T04:35:39,645 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/b/c3130e7af5254ba2bda97dd102f52c9c as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/b/c3130e7af5254ba2bda97dd102f52c9c 2024-11-14T04:35:39,650 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/b/c3130e7af5254ba2bda97dd102f52c9c, entries=10, sequenceid=35, filesize=5.8 K 2024-11-14T04:35:39,651 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/c/6b0152f6a6fb4e5287ce6567b0eca5f2 as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/c/6b0152f6a6fb4e5287ce6567b0eca5f2 2024-11-14T04:35:39,656 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/c/6b0152f6a6fb4e5287ce6567b0eca5f2, entries=10, sequenceid=35, filesize=5.8 K 2024-11-14T04:35:39,656 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 3ec537b454cc5cf85ff686857ee98810 in 62ms, sequenceid=35, compaction requested=false; wal=null 2024-11-14T04:35:39,657 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000035 2024-11-14T04:35:39,658 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,658 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,658 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:39,659 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:39,661 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-14T04:35:39,662 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3ec537b454cc5cf85ff686857ee98810; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63551464, jitterRate=-0.05300939083099365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:39,662 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3ec537b454cc5cf85ff686857ee98810: Writing region info on filesystem at 1731558939555Initializing all the Stores at 1731558939556 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939556Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939556Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558939556Obtaining lock to block concurrent updates at 1731558939594 (+38 ms)Preparing flush snapshotting stores in 3ec537b454cc5cf85ff686857ee98810 at 1731558939594Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731558939594Flushing stores of testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. at 1731558939594Flushing 3ec537b454cc5cf85ff686857ee98810/b: creating writer at 1731558939594Flushing 3ec537b454cc5cf85ff686857ee98810/b: appending metadata at 1731558939606 (+12 ms)Flushing 3ec537b454cc5cf85ff686857ee98810/b: closing flushed file at 1731558939606Flushing 3ec537b454cc5cf85ff686857ee98810/c: creating writer at 1731558939619 (+13 ms)Flushing 3ec537b454cc5cf85ff686857ee98810/c: appending metadata at 1731558939633 (+14 ms)Flushing 3ec537b454cc5cf85ff686857ee98810/c: closing flushed file at 1731558939633Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7307def3: reopening flushed file at 1731558939644 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65745023: reopening flushed file at 1731558939650 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 3ec537b454cc5cf85ff686857ee98810 in 62ms, sequenceid=35, compaction requested=false; wal=null at 1731558939656 (+6 ms)Cleaning up temporary data from old regions at 1731558939658 (+2 ms)Region opened successfully at 1731558939662 (+4 ms) 2024-11-14T04:35:39,719 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457, size=0 (0bytes) 2024-11-14T04:35:39,719 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 might be still open, length is 0 2024-11-14T04:35:39,719 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 2024-11-14T04:35:39,720 WARN [IPC Server handler 2 on default port 41731 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-14T04:35:39,720 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 after 1ms 2024-11-14T04:35:41,559 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:59648 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33533:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59648 dst: /127.0.0.1:33533 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33533 remote=/127.0.0.1:59648]. Total timeout mills is 60000, 58128 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:41,560 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:49554 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37377:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49554 dst: /127.0.0.1:37377 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:41,560 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-345787309_22 at /127.0.0.1:39898 [Receiving block BP-954332279-172.17.0.3-1731558898852:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39898 dst: /127.0.0.1:36191 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:41,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741930_1112 (size=5567) 2024-11-14T04:35:41,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741930_1112 (size=5567) 2024-11-14T04:35:43,721 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 after 4002ms 2024-11-14T04:35:43,756 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:43,756 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 took 4037ms 2024-11-14T04:35:43,758 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457; continuing. 2024-11-14T04:35:43,759 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 so closing down 2024-11-14T04:35:43,759 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T04:35:43,760 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731558939457.temp 2024-11-14T04:35:43,783 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000037-wal.1731558939457.temp 2024-11-14T04:35:43,784 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T04:35:43,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741933_1113 (size=5575) 2024-11-14T04:35:43,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741933_1113 (size=5575) 2024-11-14T04:35:43,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741933_1113 (size=5575) 2024-11-14T04:35:43,789 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000037-wal.1731558939457.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T04:35:43,790 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000037-wal.1731558939457.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000066 2024-11-14T04:35:43,790 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T04:35:43,790 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457, journal: Splitting hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457, size=0 (0bytes) at 1731558939719Finishing writing output for hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 so closing down at 1731558943759 (+4040 ms)Creating recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000037-wal.1731558939457.temp at 1731558943783 (+24 ms)3 split writer threads finished at 1731558943784 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000037-wal.1731558939457.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731558943789 (+5 ms)Rename recovered edits hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000037-wal.1731558939457.temp to hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000066 at 1731558943790 (+1 ms)Processed 30 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457, size=0, length=0, corrupted=false, cancelled=false at 1731558943790 2024-11-14T04:35:43,792 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 to hdfs://localhost:41731/hbase/oldWALs/wal.1731558939457 2024-11-14T04:35:43,792 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000066 2024-11-14T04:35:43,792 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T04:35:43,794 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41731/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177, archiveDir=hdfs://localhost:41731/hbase/oldWALs, maxLogs=32 2024-11-14T04:35:43,806 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558943794, exclude list is [], retry=0 2024-11-14T04:35:43,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37377,DS-6d4a911c-8872-4483-b251-f501bed14cdd,DISK] 2024-11-14T04:35:43,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36191,DS-70d64bb0-53e7-483c-982e-2c826e578c07,DISK] 2024-11-14T04:35:43,809 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33533,DS-73f7b8f5-1aec-482c-b4a3-0eedd4b2b69b,DISK] 2024-11-14T04:35:43,834 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558943794 2024-11-14T04:35:43,834 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44105:44105),(127.0.0.1/127.0.0.1:42129:42129),(127.0.0.1/127.0.0.1:46855:46855)] 2024-11-14T04:35:43,834 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T04:35:43,836 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:43,837 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName a 2024-11-14T04:35:43,837 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:43,842 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/a/5edb967245bb4ef2bfa1bfc775a9e51b 2024-11-14T04:35:43,842 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:43,842 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:43,843 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName b 2024-11-14T04:35:43,843 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:43,849 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/b/c3130e7af5254ba2bda97dd102f52c9c 2024-11-14T04:35:43,849 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:43,849 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:43,850 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ec537b454cc5cf85ff686857ee98810 columnFamilyName c 2024-11-14T04:35:43,850 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T04:35:43,856 DEBUG [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/c/6b0152f6a6fb4e5287ce6567b0eca5f2 2024-11-14T04:35:43,856 INFO [StoreOpener-3ec537b454cc5cf85ff686857ee98810-1 {}] regionserver.HStore(327): Store=3ec537b454cc5cf85ff686857ee98810/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T04:35:43,856 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:43,857 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:43,858 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:43,859 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000066 2024-11-14T04:35:43,885 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-14T04:35:43,889 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000066 2024-11-14T04:35:43,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-14T04:35:43,889 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3ec537b454cc5cf85ff686857ee98810 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-14T04:35:43,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-14T04:35:43,890 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-14T04:35:43,890 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-14T04:35:43,908 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/a/1e449968ce3547538ade2b9387e116eb is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731558939669/Put/seqid=0 2024-11-14T04:35:43,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741935_1115 (size=5958) 2024-11-14T04:35:43,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741935_1115 (size=5958) 2024-11-14T04:35:43,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741935_1115 (size=5958) 2024-11-14T04:35:43,915 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/a/1e449968ce3547538ade2b9387e116eb 2024-11-14T04:35:43,935 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/b/23322dfc1de442408c79f064d8d06cbc is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731558939675/Put/seqid=0 2024-11-14T04:35:43,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741936_1116 (size=5958) 2024-11-14T04:35:43,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741936_1116 (size=5958) 2024-11-14T04:35:43,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741936_1116 (size=5958) 2024-11-14T04:35:43,943 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/b/23322dfc1de442408c79f064d8d06cbc 2024-11-14T04:35:43,962 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/c/68ddd3b5991642a7b5a80fd1be190b7e is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731558939681/Put/seqid=0 2024-11-14T04:35:43,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741937_1117 (size=5958) 2024-11-14T04:35:43,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741937_1117 (size=5958) 2024-11-14T04:35:43,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741937_1117 (size=5958) 2024-11-14T04:35:43,969 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/c/68ddd3b5991642a7b5a80fd1be190b7e 2024-11-14T04:35:43,975 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/a/1e449968ce3547538ade2b9387e116eb as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/a/1e449968ce3547538ade2b9387e116eb 2024-11-14T04:35:43,981 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/a/1e449968ce3547538ade2b9387e116eb, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T04:35:43,982 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/b/23322dfc1de442408c79f064d8d06cbc as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/b/23322dfc1de442408c79f064d8d06cbc 2024-11-14T04:35:43,988 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/b/23322dfc1de442408c79f064d8d06cbc, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T04:35:43,989 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/.tmp/c/68ddd3b5991642a7b5a80fd1be190b7e as hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/c/68ddd3b5991642a7b5a80fd1be190b7e 2024-11-14T04:35:43,996 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/c/68ddd3b5991642a7b5a80fd1be190b7e, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T04:35:43,996 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 3ec537b454cc5cf85ff686857ee98810 in 107ms, sequenceid=66, compaction requested=false; wal=null 2024-11-14T04:35:44,000 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/0000000000000000066 2024-11-14T04:35:44,001 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:44,001 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:44,003 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T04:35:44,004 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3ec537b454cc5cf85ff686857ee98810 2024-11-14T04:35:44,006 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/hbase/data/default/testReplayEditsWrittenViaHRegion/3ec537b454cc5cf85ff686857ee98810/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-14T04:35:44,007 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3ec537b454cc5cf85ff686857ee98810; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65309636, jitterRate=-0.02681058645248413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T04:35:44,008 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3ec537b454cc5cf85ff686857ee98810: Writing region info on filesystem at 1731558943835Initializing all the Stores at 1731558943835Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558943835Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558943836 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731558943836Obtaining lock to block concurrent updates at 1731558943889 (+53 ms)Preparing flush snapshotting stores in 3ec537b454cc5cf85ff686857ee98810 at 1731558943889Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731558943889Flushing stores of testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. at 1731558943889Flushing 3ec537b454cc5cf85ff686857ee98810/a: creating writer at 1731558943889Flushing 3ec537b454cc5cf85ff686857ee98810/a: appending metadata at 1731558943907 (+18 ms)Flushing 3ec537b454cc5cf85ff686857ee98810/a: closing flushed file at 1731558943907Flushing 3ec537b454cc5cf85ff686857ee98810/b: creating writer at 1731558943919 (+12 ms)Flushing 3ec537b454cc5cf85ff686857ee98810/b: appending metadata at 1731558943934 (+15 ms)Flushing 3ec537b454cc5cf85ff686857ee98810/b: closing flushed file at 1731558943934Flushing 3ec537b454cc5cf85ff686857ee98810/c: creating writer at 1731558943948 (+14 ms)Flushing 3ec537b454cc5cf85ff686857ee98810/c: appending metadata at 1731558943962 (+14 ms)Flushing 3ec537b454cc5cf85ff686857ee98810/c: closing flushed file at 1731558943962Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@225debdd: reopening flushed file at 1731558943974 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a68441f: reopening flushed file at 1731558943981 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a6c5ef3: reopening flushed file at 1731558943988 (+7 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 3ec537b454cc5cf85ff686857ee98810 in 107ms, sequenceid=66, compaction requested=false; wal=null at 1731558943996 (+8 ms)Cleaning up temporary data from old regions at 1731558944001 (+5 ms)Region opened successfully at 1731558944007 (+6 ms) 2024-11-14T04:35:44,022 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3ec537b454cc5cf85ff686857ee98810, disabling compactions & flushes 2024-11-14T04:35:44,022 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:44,022 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:44,022 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. after waiting 0 ms 2024-11-14T04:35:44,022 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:44,023 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731558939178.3ec537b454cc5cf85ff686857ee98810. 2024-11-14T04:35:44,024 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3ec537b454cc5cf85ff686857ee98810: Waiting for close lock at 1731558944022Disabling compacts and flushes for region at 1731558944022Disabling writes for close at 1731558944022Writing region close event to WAL at 1731558944023 (+1 ms)Closed at 1731558944023 2024-11-14T04:35:44,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741934_1114 (size=175) 2024-11-14T04:35:44,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741934_1114 (size=175) 2024-11-14T04:35:44,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741934_1114 (size=175) 2024-11-14T04:35:44,029 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T04:35:44,029 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731558943794) 2024-11-14T04:35:44,044 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444 (was 438) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1750902780_22 at /127.0.0.1:40862 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:41731 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1750902780_22 at /127.0.0.1:39520 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:41731 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1750902780_22 at /127.0.0.1:48374 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=276 (was 300), ProcessCount=11 (was 11), AvailableMemoryMB=6623 (was 6639) 2024-11-14T04:35:44,045 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-14T04:35:44,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T04:35:44,045 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T04:35:44,045 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:35:44,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:35:44,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:35:44,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T04:35:44,046 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T04:35:44,046 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1263205630, stopped=false 2024-11-14T04:35:44,047 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b024da739ae8,36617,1731558903563 2024-11-14T04:35:44,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T04:35:44,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T04:35:44,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T04:35:44,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:44,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:44,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:44,122 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T04:35:44,122 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T04:35:44,122 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:35:44,122 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:35:44,122 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:35:44,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:35:44,123 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b024da739ae8,42805,1731558904393' ***** 2024-11-14T04:35:44,123 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T04:35:44,123 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T04:35:44,123 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b024da739ae8,38767,1731558904446' ***** 2024-11-14T04:35:44,123 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T04:35:44,123 INFO [RS:2;b024da739ae8:38767 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T04:35:44,123 INFO [RS:2;b024da739ae8:38767 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T04:35:44,123 INFO [RS:2;b024da739ae8:38767 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T04:35:44,123 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T04:35:44,123 INFO [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(959): stopping server b024da739ae8,38767,1731558904446 2024-11-14T04:35:44,123 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T04:35:44,124 INFO [RS:2;b024da739ae8:38767 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:35:44,124 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T04:35:44,124 INFO [RS:2;b024da739ae8:38767 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;b024da739ae8:38767. 2024-11-14T04:35:44,124 INFO [RS:1;b024da739ae8:42805 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T04:35:44,124 DEBUG [RS:2;b024da739ae8:38767 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:35:44,124 INFO [RS:1;b024da739ae8:42805 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T04:35:44,124 DEBUG [RS:2;b024da739ae8:38767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:35:44,124 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(3091): Received CLOSE for 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:44,124 INFO [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(976): stopping server b024da739ae8,38767,1731558904446; all regions closed. 2024-11-14T04:35:44,124 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(959): stopping server b024da739ae8,42805,1731558904393 2024-11-14T04:35:44,124 INFO [RS:1;b024da739ae8:42805 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:35:44,124 INFO [RS:1;b024da739ae8:42805 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b024da739ae8:42805. 2024-11-14T04:35:44,124 DEBUG [RS:1;b024da739ae8:42805 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T04:35:44,124 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 34ae018529a53a855c7af76d8741b2e3, disabling compactions & flushes 2024-11-14T04:35:44,124 DEBUG [RS:1;b024da739ae8:42805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:35:44,124 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:44,124 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:44,124 INFO [RS:1;b024da739ae8:42805 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T04:35:44,124 INFO [RS:1;b024da739ae8:42805 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T04:35:44,124 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. after waiting 0 ms 2024-11-14T04:35:44,125 INFO [RS:1;b024da739ae8:42805 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T04:35:44,125 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:44,125 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T04:35:44,125 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T04:35:44,125 DEBUG [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(1325): Online Regions={34ae018529a53a855c7af76d8741b2e3=testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3., 1588230740=hbase:meta,,1.1588230740} 2024-11-14T04:35:44,125 DEBUG [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 34ae018529a53a855c7af76d8741b2e3 2024-11-14T04:35:44,125 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T04:35:44,125 INFO [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T04:35:44,125 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T04:35:44,125 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T04:35:44,125 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T04:35:44,125 INFO [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-14T04:35:44,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741835_1011 (size=175) 2024-11-14T04:35:44,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741835_1011 (size=175) 2024-11-14T04:35:44,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741835_1011 (size=175) 2024-11-14T04:35:44,130 DEBUG [RS:2;b024da739ae8:38767 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/oldWALs 2024-11-14T04:35:44,130 INFO [RS:2;b024da739ae8:38767 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b024da739ae8%2C38767%2C1731558904446:(num 1731558905917) 2024-11-14T04:35:44,130 DEBUG [RS:2;b024da739ae8:38767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:35:44,130 INFO [RS:2;b024da739ae8:38767 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:35:44,130 INFO [RS:2;b024da739ae8:38767 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:35:44,131 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/default/testReplayEditsAfterRegionMovedWithMultiCF/34ae018529a53a855c7af76d8741b2e3/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-14T04:35:44,131 INFO [RS:2;b024da739ae8:38767 {}] hbase.ChoreService(370): Chore service for: regionserver/b024da739ae8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T04:35:44,131 INFO [RS:2;b024da739ae8:38767 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T04:35:44,131 INFO [RS:2;b024da739ae8:38767 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T04:35:44,131 INFO [RS:2;b024da739ae8:38767 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T04:35:44,131 INFO [RS:2;b024da739ae8:38767 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:35:44,131 INFO [RS:2;b024da739ae8:38767 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38767 2024-11-14T04:35:44,131 INFO [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:44,131 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 34ae018529a53a855c7af76d8741b2e3: Waiting for close lock at 1731558944124Running coprocessor pre-close hooks at 1731558944124Disabling compacts and flushes for region at 1731558944124Disabling writes for close at 1731558944124Writing region close event to WAL at 1731558944127 (+3 ms)Running coprocessor post-close hooks at 1731558944131 (+4 ms)Closed at 1731558944131 2024-11-14T04:35:44,131 INFO [regionserver/b024da739ae8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:35:44,131 DEBUG [RS_CLOSE_REGION-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3. 2024-11-14T04:35:44,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:35:44,142 INFO [RS:2;b024da739ae8:38767 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:35:44,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b024da739ae8,38767,1731558904446 2024-11-14T04:35:44,151 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/.tmp/info/3d91975de1604398a8bf3bb47b35bec5 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731558919729.34ae018529a53a855c7af76d8741b2e3./info:regioninfo/1731558923360/Put/seqid=0 2024-11-14T04:35:44,153 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b024da739ae8,38767,1731558904446] 2024-11-14T04:35:44,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741938_1118 (size=8243) 2024-11-14T04:35:44,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741938_1118 (size=8243) 2024-11-14T04:35:44,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741938_1118 (size=8243) 2024-11-14T04:35:44,158 INFO [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/.tmp/info/3d91975de1604398a8bf3bb47b35bec5 2024-11-14T04:35:44,163 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b024da739ae8,38767,1731558904446 already deleted, retry=false 2024-11-14T04:35:44,163 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b024da739ae8,38767,1731558904446 expired; onlineServers=1 2024-11-14T04:35:44,181 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/.tmp/ns/f9252a07582a4ac2a07771d5864094cf is 43, key is default/ns:d/1731558906625/Put/seqid=0 2024-11-14T04:35:44,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741939_1119 (size=5153) 2024-11-14T04:35:44,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741939_1119 (size=5153) 2024-11-14T04:35:44,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741939_1119 (size=5153) 2024-11-14T04:35:44,188 INFO [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/.tmp/ns/f9252a07582a4ac2a07771d5864094cf 2024-11-14T04:35:44,206 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/.tmp/table/4fe40e83c7224f29b4c1cf46571cb405 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731558920148/Put/seqid=0 2024-11-14T04:35:44,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741940_1120 (size=5431) 2024-11-14T04:35:44,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741940_1120 (size=5431) 2024-11-14T04:35:44,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741940_1120 (size=5431) 2024-11-14T04:35:44,212 INFO [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/.tmp/table/4fe40e83c7224f29b4c1cf46571cb405 2024-11-14T04:35:44,214 INFO [regionserver/b024da739ae8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:35:44,218 INFO [regionserver/b024da739ae8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:35:44,219 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/.tmp/info/3d91975de1604398a8bf3bb47b35bec5 as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/info/3d91975de1604398a8bf3bb47b35bec5 2024-11-14T04:35:44,223 INFO [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/info/3d91975de1604398a8bf3bb47b35bec5, entries=18, sequenceid=21, filesize=8.0 K 2024-11-14T04:35:44,224 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/.tmp/ns/f9252a07582a4ac2a07771d5864094cf as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/ns/f9252a07582a4ac2a07771d5864094cf 2024-11-14T04:35:44,229 INFO [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/ns/f9252a07582a4ac2a07771d5864094cf, entries=2, sequenceid=21, filesize=5.0 K 2024-11-14T04:35:44,230 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/.tmp/table/4fe40e83c7224f29b4c1cf46571cb405 as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/table/4fe40e83c7224f29b4c1cf46571cb405 2024-11-14T04:35:44,235 INFO [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/table/4fe40e83c7224f29b4c1cf46571cb405, entries=2, sequenceid=21, filesize=5.3 K 2024-11-14T04:35:44,236 INFO [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=21, compaction requested=false 2024-11-14T04:35:44,240 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-14T04:35:44,240 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T04:35:44,241 INFO [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T04:35:44,241 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731558944125Running coprocessor pre-close hooks at 1731558944125Disabling compacts and flushes for region at 1731558944125Disabling writes for close at 1731558944125Obtaining lock to block concurrent updates at 1731558944125Preparing flush snapshotting stores in 1588230740 at 1731558944125Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731558944126 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731558944127 (+1 ms)Flushing 1588230740/info: creating writer at 1731558944128 (+1 ms)Flushing 1588230740/info: appending metadata at 1731558944151 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731558944151Flushing 1588230740/ns: creating writer at 1731558944165 (+14 ms)Flushing 1588230740/ns: appending metadata at 1731558944181 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731558944181Flushing 1588230740/table: creating writer at 1731558944193 (+12 ms)Flushing 1588230740/table: appending metadata at 1731558944206 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731558944206Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2af84c08: reopening flushed file at 1731558944218 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6292d7a: reopening flushed file at 1731558944224 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@681788d1: reopening flushed file at 1731558944229 (+5 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=21, compaction requested=false at 1731558944236 (+7 ms)Writing region close event to WAL at 1731558944237 (+1 ms)Running coprocessor post-close hooks at 1731558944240 (+3 ms)Closed at 1731558944240 2024-11-14T04:35:44,241 DEBUG [RS_CLOSE_META-regionserver/b024da739ae8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T04:35:44,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:35:44,253 INFO [RS:2;b024da739ae8:38767 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:35:44,253 INFO [RS:2;b024da739ae8:38767 {}] regionserver.HRegionServer(1031): Exiting; stopping=b024da739ae8,38767,1731558904446; zookeeper connection closed. 2024-11-14T04:35:44,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38767-0x10137a0072b0003, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:35:44,253 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@417e00e3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@417e00e3 2024-11-14T04:35:44,325 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(976): stopping server b024da739ae8,42805,1731558904393; all regions closed. 2024-11-14T04:35:44,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741836_1012 (size=9487) 2024-11-14T04:35:44,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741836_1012 (size=9487) 2024-11-14T04:35:44,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741836_1012 (size=9487) 2024-11-14T04:35:44,331 DEBUG [RS:1;b024da739ae8:42805 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/oldWALs 2024-11-14T04:35:44,331 INFO [RS:1;b024da739ae8:42805 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b024da739ae8%2C42805%2C1731558904393.meta:.meta(num 1731558906371) 2024-11-14T04:35:44,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741833_1009 (size=3621) 2024-11-14T04:35:44,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741833_1009 (size=3621) 2024-11-14T04:35:44,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741833_1009 (size=3621) 2024-11-14T04:35:44,336 DEBUG [RS:1;b024da739ae8:42805 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/oldWALs 2024-11-14T04:35:44,337 INFO [RS:1;b024da739ae8:42805 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b024da739ae8%2C42805%2C1731558904393:(num 1731558905917) 2024-11-14T04:35:44,337 DEBUG [RS:1;b024da739ae8:42805 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T04:35:44,337 INFO [RS:1;b024da739ae8:42805 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T04:35:44,337 INFO [RS:1;b024da739ae8:42805 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:35:44,337 INFO [RS:1;b024da739ae8:42805 {}] hbase.ChoreService(370): Chore service for: regionserver/b024da739ae8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T04:35:44,337 INFO [RS:1;b024da739ae8:42805 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:35:44,337 INFO [regionserver/b024da739ae8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:35:44,337 INFO [RS:1;b024da739ae8:42805 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42805 2024-11-14T04:35:44,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T04:35:44,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b024da739ae8,42805,1731558904393 2024-11-14T04:35:44,342 INFO [RS:1;b024da739ae8:42805 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:35:44,352 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b024da739ae8,42805,1731558904393] 2024-11-14T04:35:44,363 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b024da739ae8,42805,1731558904393 already deleted, retry=false 2024-11-14T04:35:44,363 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b024da739ae8,42805,1731558904393 expired; onlineServers=0 2024-11-14T04:35:44,363 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b024da739ae8,36617,1731558903563' ***** 2024-11-14T04:35:44,363 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T04:35:44,364 INFO [M:0;b024da739ae8:36617 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T04:35:44,364 INFO [M:0;b024da739ae8:36617 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T04:35:44,364 DEBUG [M:0;b024da739ae8:36617 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T04:35:44,364 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T04:35:44,364 DEBUG [M:0;b024da739ae8:36617 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T04:35:44,364 DEBUG [master/b024da739ae8:0:becomeActiveMaster-HFileCleaner.large.0-1731558905594 {}] cleaner.HFileCleaner(306): Exit Thread[master/b024da739ae8:0:becomeActiveMaster-HFileCleaner.large.0-1731558905594,5,FailOnTimeoutGroup] 2024-11-14T04:35:44,364 DEBUG [master/b024da739ae8:0:becomeActiveMaster-HFileCleaner.small.0-1731558905595 {}] cleaner.HFileCleaner(306): Exit Thread[master/b024da739ae8:0:becomeActiveMaster-HFileCleaner.small.0-1731558905595,5,FailOnTimeoutGroup] 2024-11-14T04:35:44,365 INFO [M:0;b024da739ae8:36617 {}] hbase.ChoreService(370): Chore service for: master/b024da739ae8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T04:35:44,365 INFO [M:0;b024da739ae8:36617 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T04:35:44,365 DEBUG [M:0;b024da739ae8:36617 {}] master.HMaster(1795): Stopping service threads 2024-11-14T04:35:44,365 INFO [M:0;b024da739ae8:36617 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T04:35:44,365 INFO [M:0;b024da739ae8:36617 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T04:35:44,366 INFO [M:0;b024da739ae8:36617 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T04:35:44,367 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T04:35:44,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T04:35:44,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T04:35:44,374 DEBUG [M:0;b024da739ae8:36617 {}] zookeeper.ZKUtil(347): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T04:35:44,374 WARN [M:0;b024da739ae8:36617 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T04:35:44,375 INFO [M:0;b024da739ae8:36617 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/.lastflushedseqids 2024-11-14T04:35:44,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741941_1121 (size=138) 2024-11-14T04:35:44,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741941_1121 (size=138) 2024-11-14T04:35:44,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741941_1121 (size=138) 2024-11-14T04:35:44,390 INFO [M:0;b024da739ae8:36617 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T04:35:44,390 INFO [M:0;b024da739ae8:36617 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T04:35:44,391 DEBUG [M:0;b024da739ae8:36617 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T04:35:44,391 INFO [M:0;b024da739ae8:36617 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:35:44,391 DEBUG [M:0;b024da739ae8:36617 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:35:44,391 DEBUG [M:0;b024da739ae8:36617 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T04:35:44,391 DEBUG [M:0;b024da739ae8:36617 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:35:44,391 INFO [M:0;b024da739ae8:36617 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-14T04:35:44,407 DEBUG [M:0;b024da739ae8:36617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f643842e2aa94fcbb3019e1823adb73e is 82, key is hbase:meta,,1/info:regioninfo/1731558906479/Put/seqid=0 2024-11-14T04:35:44,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741942_1122 (size=5672) 2024-11-14T04:35:44,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741942_1122 (size=5672) 2024-11-14T04:35:44,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741942_1122 (size=5672) 2024-11-14T04:35:44,413 INFO [M:0;b024da739ae8:36617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f643842e2aa94fcbb3019e1823adb73e 2024-11-14T04:35:44,431 DEBUG [M:0;b024da739ae8:36617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0cecc1fe58874378a6e6d679b76f568c is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731558920152/Put/seqid=0 2024-11-14T04:35:44,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741943_1123 (size=7755) 2024-11-14T04:35:44,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741943_1123 (size=7755) 2024-11-14T04:35:44,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741943_1123 (size=7755) 2024-11-14T04:35:44,443 INFO [M:0;b024da739ae8:36617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0cecc1fe58874378a6e6d679b76f568c 2024-11-14T04:35:44,448 INFO [M:0;b024da739ae8:36617 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0cecc1fe58874378a6e6d679b76f568c 2024-11-14T04:35:44,453 INFO [RS:1;b024da739ae8:42805 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:35:44,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:35:44,453 INFO [RS:1;b024da739ae8:42805 {}] regionserver.HRegionServer(1031): Exiting; stopping=b024da739ae8,42805,1731558904393; zookeeper connection closed. 2024-11-14T04:35:44,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42805-0x10137a0072b0002, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:35:44,453 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1110caa5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1110caa5 2024-11-14T04:35:44,453 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-14T04:35:44,469 DEBUG [M:0;b024da739ae8:36617 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/707ea5e5db1444f2852220bb9190a224 is 69, key is b024da739ae8,38767,1731558904446/rs:state/1731558905630/Put/seqid=0 2024-11-14T04:35:44,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741944_1124 (size=5440) 2024-11-14T04:35:44,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741944_1124 (size=5440) 2024-11-14T04:35:44,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741944_1124 (size=5440) 2024-11-14T04:35:44,477 INFO [M:0;b024da739ae8:36617 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/707ea5e5db1444f2852220bb9190a224 2024-11-14T04:35:44,482 INFO [M:0;b024da739ae8:36617 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 707ea5e5db1444f2852220bb9190a224 2024-11-14T04:35:44,483 DEBUG [M:0;b024da739ae8:36617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f643842e2aa94fcbb3019e1823adb73e as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f643842e2aa94fcbb3019e1823adb73e 2024-11-14T04:35:44,489 INFO [M:0;b024da739ae8:36617 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f643842e2aa94fcbb3019e1823adb73e, entries=8, sequenceid=168, filesize=5.5 K 2024-11-14T04:35:44,490 DEBUG [M:0;b024da739ae8:36617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0cecc1fe58874378a6e6d679b76f568c as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0cecc1fe58874378a6e6d679b76f568c 2024-11-14T04:35:44,495 INFO [M:0;b024da739ae8:36617 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0cecc1fe58874378a6e6d679b76f568c 2024-11-14T04:35:44,495 INFO [M:0;b024da739ae8:36617 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0cecc1fe58874378a6e6d679b76f568c, entries=17, sequenceid=168, filesize=7.6 K 2024-11-14T04:35:44,496 DEBUG [M:0;b024da739ae8:36617 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/707ea5e5db1444f2852220bb9190a224 as hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/707ea5e5db1444f2852220bb9190a224 2024-11-14T04:35:44,501 INFO [M:0;b024da739ae8:36617 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 707ea5e5db1444f2852220bb9190a224 2024-11-14T04:35:44,501 INFO [M:0;b024da739ae8:36617 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41731/user/jenkins/test-data/696f05ee-e6b3-6183-1c2f-a8620581e0bc/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/707ea5e5db1444f2852220bb9190a224, entries=3, sequenceid=168, filesize=5.3 K 2024-11-14T04:35:44,502 INFO [M:0;b024da739ae8:36617 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=168, compaction requested=false 2024-11-14T04:35:44,503 INFO [M:0;b024da739ae8:36617 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T04:35:44,503 DEBUG [M:0;b024da739ae8:36617 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731558944391Disabling compacts and flushes for region at 1731558944391Disabling writes for close at 1731558944391Obtaining lock to block concurrent updates at 1731558944391Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731558944391Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1731558944391Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731558944392 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731558944392Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731558944407 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731558944407Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731558944417 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731558944431 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731558944431Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731558944448 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731558944468 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731558944468Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fb3f536: reopening flushed file at 1731558944482 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19abfcf6: reopening flushed file at 1731558944489 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65ef2500: reopening flushed file at 1731558944495 (+6 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 111ms, sequenceid=168, compaction requested=false at 1731558944502 (+7 ms)Writing region close event to WAL at 1731558944503 (+1 ms)Closed at 1731558944503 2024-11-14T04:35:44,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33533 is added to blk_1073741830_1006 (size=83606) 2024-11-14T04:35:44,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37377 is added to blk_1073741830_1006 (size=83606) 2024-11-14T04:35:44,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36191 is added to blk_1073741830_1006 (size=83606) 2024-11-14T04:35:44,506 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T04:35:44,506 INFO [M:0;b024da739ae8:36617 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T04:35:44,506 INFO [M:0;b024da739ae8:36617 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36617 2024-11-14T04:35:44,506 INFO [M:0;b024da739ae8:36617 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T04:35:44,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:35:44,632 INFO [M:0;b024da739ae8:36617 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T04:35:44,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36617-0x10137a0072b0000, quorum=127.0.0.1:51455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T04:35:44,637 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923978 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923978 (inode 16655) Holder DFSClient_NONMAPREDUCE_-345787309_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731558923623/wal.1731558923978 (inode 16655) Holder DFSClient_NONMAPREDUCE_-345787309_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T04:35:44,638 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731558915228/wal.1731558915337 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:44,639 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731558939177/wal.1731558939457 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:44,641 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558933518 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558933518 (inode 16678) Holder DFSClient_NONMAPREDUCE_-345787309_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731558924167/wal.1731558933518 (inode 16678) Holder DFSClient_NONMAPREDUCE_-345787309_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T04:35:44,641 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731558908141/wal.1731558908229 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:44,644 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558934388 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558934388 (inode 16704) Holder DFSClient_NONMAPREDUCE_-345787309_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731558933783/wal.1731558934388 (inode 16704) Holder DFSClient_NONMAPREDUCE_-345787309_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T04:35:44,644 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731558934622/wal.1731558934706 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T04:35:44,646 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal.1731558907958 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal.1731558907958 (inode 16485) Holder DFSClient_NONMAPREDUCE_-345787309_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731558907584/wal.1731558907958 (inode 16485) Holder DFSClient_NONMAPREDUCE_-345787309_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T04:35:44,649 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal.1731558907312 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal.1731558907312 (inode 16462) Holder DFSClient_NONMAPREDUCE_-345787309_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731558906886/wal.1731558907312 (inode 16462) Holder DFSClient_NONMAPREDUCE_-345787309_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T04:35:44,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T04:35:44,655 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T04:35:44,655 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T04:35:44,655 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T04:35:44,656 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/hadoop.log.dir/,STOPPED} 2024-11-14T04:35:44,660 WARN [BP-954332279-172.17.0.3-1731558898852 heartbeating to localhost/127.0.0.1:41731 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T04:35:44,660 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T04:35:44,660 WARN [BP-954332279-172.17.0.3-1731558898852 heartbeating to localhost/127.0.0.1:41731 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-954332279-172.17.0.3-1731558898852 (Datanode Uuid 69d0a3f2-cc3e-4761-920d-3b67f21672d1) service to localhost/127.0.0.1:41731 2024-11-14T04:35:44,660 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T04:35:44,661 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data5/current/BP-954332279-172.17.0.3-1731558898852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T04:35:44,661 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data6/current/BP-954332279-172.17.0.3-1731558898852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T04:35:44,662 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T04:35:44,664 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T04:35:44,664 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T04:35:44,664 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T04:35:44,664 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T04:35:44,664 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/hadoop.log.dir/,STOPPED} 2024-11-14T04:35:44,665 WARN [BP-954332279-172.17.0.3-1731558898852 heartbeating to localhost/127.0.0.1:41731 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T04:35:44,665 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T04:35:44,665 WARN [BP-954332279-172.17.0.3-1731558898852 heartbeating to localhost/127.0.0.1:41731 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-954332279-172.17.0.3-1731558898852 (Datanode Uuid 6b000cc2-4eed-412d-9988-f74dc86d9c91) service to localhost/127.0.0.1:41731 2024-11-14T04:35:44,665 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T04:35:44,666 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data3/current/BP-954332279-172.17.0.3-1731558898852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T04:35:44,666 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data4/current/BP-954332279-172.17.0.3-1731558898852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T04:35:44,666 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T04:35:44,668 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T04:35:44,668 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T04:35:44,668 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T04:35:44,669 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T04:35:44,669 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/hadoop.log.dir/,STOPPED} 2024-11-14T04:35:44,670 WARN [BP-954332279-172.17.0.3-1731558898852 heartbeating to localhost/127.0.0.1:41731 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T04:35:44,670 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T04:35:44,670 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T04:35:44,670 WARN [BP-954332279-172.17.0.3-1731558898852 heartbeating to localhost/127.0.0.1:41731 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-954332279-172.17.0.3-1731558898852 (Datanode Uuid 337cb191-b596-474b-8076-6ace53fb5e5c) service to localhost/127.0.0.1:41731 2024-11-14T04:35:44,670 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data1/current/BP-954332279-172.17.0.3-1731558898852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T04:35:44,671 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/cluster_f7869f5b-88a3-b63c-c11e-f7897828f708/data/data2/current/BP-954332279-172.17.0.3-1731558898852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T04:35:44,671 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T04:35:44,676 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T04:35:44,677 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T04:35:44,677 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T04:35:44,677 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T04:35:44,677 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0191368a-eedb-e464-b493-b50678c45c1e/hadoop.log.dir/,STOPPED} 2024-11-14T04:35:44,686 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T04:35:44,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down