2024-12-01 23:28:06,578 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 2024-12-01 23:28:06,621 main DEBUG Took 0.040105 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-01 23:28:06,622 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-01 23:28:06,622 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-01 23:28:06,624 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-01 23:28:06,636 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,649 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-01 23:28:06,682 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,686 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,687 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,688 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,688 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,689 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,691 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,693 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,698 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,698 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,699 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,700 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,701 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,701 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,703 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,704 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,705 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,706 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,707 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,708 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 23:28:06,709 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,709 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-01 23:28:06,711 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 23:28:06,713 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-01 23:28:06,716 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-01 23:28:06,717 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-01 23:28:06,719 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-01 23:28:06,720 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-01 23:28:06,733 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-01 23:28:06,738 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-01 23:28:06,740 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-01 23:28:06,740 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-01 23:28:06,741 main DEBUG createAppenders(={Console}) 2024-12-01 23:28:06,742 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 initialized 2024-12-01 23:28:06,743 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 2024-12-01 23:28:06,743 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7857fe2 OK. 2024-12-01 23:28:06,744 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-01 23:28:06,745 main DEBUG OutputStream closed 2024-12-01 23:28:06,745 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-01 23:28:06,745 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-01 23:28:06,746 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@54a7079e OK 2024-12-01 23:28:06,886 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-01 23:28:06,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-01 23:28:06,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-01 23:28:06,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-01 23:28:06,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-01 23:28:06,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-01 23:28:06,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-01 23:28:06,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-01 23:28:06,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-01 23:28:06,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-01 23:28:06,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-01 23:28:06,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-01 23:28:06,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-01 23:28:06,897 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-01 23:28:06,897 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-01 23:28:06,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-01 23:28:06,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-01 23:28:06,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-01 23:28:06,902 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01 23:28:06,903 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19b93fa8) with optional ClassLoader: null 2024-12-01 23:28:06,903 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-01 23:28:06,904 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19b93fa8] started OK. 2024-12-01T23:28:07,250 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72 2024-12-01 23:28:07,263 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-01 23:28:07,263 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01T23:28:07,279 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-01T23:28:07,286 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-01T23:28:07,320 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-01T23:28:07,385 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-01T23:28:07,385 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-01T23:28:07,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-01T23:28:07,427 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05, deleteOnExit=true 2024-12-01T23:28:07,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-01T23:28:07,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/test.cache.data in system properties and HBase conf 2024-12-01T23:28:07,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/hadoop.tmp.dir in system properties and HBase conf 2024-12-01T23:28:07,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/hadoop.log.dir in system properties and HBase conf 2024-12-01T23:28:07,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-01T23:28:07,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-01T23:28:07,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-01T23:28:07,551 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-01T23:28:07,669 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T23:28:07,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-01T23:28:07,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-01T23:28:07,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-01T23:28:07,676 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T23:28:07,676 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-01T23:28:07,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-01T23:28:07,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T23:28:07,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T23:28:07,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-01T23:28:07,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/nfs.dump.dir in system properties and HBase conf 2024-12-01T23:28:07,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/java.io.tmpdir in system properties and HBase conf 2024-12-01T23:28:07,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T23:28:07,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-01T23:28:07,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-01T23:28:08,992 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-01T23:28:09,101 INFO [Time-limited test {}] log.Log(170): Logging initialized @4177ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-01T23:28:09,215 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T23:28:09,317 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T23:28:09,389 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T23:28:09,389 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T23:28:09,392 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T23:28:09,452 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T23:28:09,463 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6626122{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/hadoop.log.dir/,AVAILABLE} 2024-12-01T23:28:09,464 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e2d6db6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T23:28:09,748 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@18f69b7e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/java.io.tmpdir/jetty-localhost-46121-hadoop-hdfs-3_4_1-tests_jar-_-any-10929339910041727561/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-01T23:28:09,763 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@51bb6da6{HTTP/1.1, (http/1.1)}{localhost:46121} 2024-12-01T23:28:09,763 INFO [Time-limited test {}] server.Server(415): Started @4840ms 2024-12-01T23:28:10,626 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T23:28:10,637 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T23:28:10,644 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T23:28:10,645 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T23:28:10,645 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T23:28:10,649 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d8adca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/hadoop.log.dir/,AVAILABLE} 2024-12-01T23:28:10,650 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a482a5c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T23:28:10,803 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@278d3ff6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/java.io.tmpdir/jetty-localhost-36755-hadoop-hdfs-3_4_1-tests_jar-_-any-18300754264994739938/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T23:28:10,804 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@b03447{HTTP/1.1, (http/1.1)}{localhost:36755} 2024-12-01T23:28:10,804 INFO [Time-limited test {}] server.Server(415): Started @5881ms 2024-12-01T23:28:10,873 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T23:28:11,103 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T23:28:11,115 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T23:28:11,141 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T23:28:11,141 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T23:28:11,141 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T23:28:11,146 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34eaca1d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/hadoop.log.dir/,AVAILABLE} 2024-12-01T23:28:11,146 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2433eef4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T23:28:11,287 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@58fd25ac{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/java.io.tmpdir/jetty-localhost-41509-hadoop-hdfs-3_4_1-tests_jar-_-any-18175928366207417236/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T23:28:11,293 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c7bf474{HTTP/1.1, (http/1.1)}{localhost:41509} 2024-12-01T23:28:11,295 INFO [Time-limited test {}] server.Server(415): Started @6371ms 2024-12-01T23:28:11,298 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T23:28:11,406 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T23:28:11,414 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T23:28:11,421 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T23:28:11,421 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T23:28:11,422 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T23:28:11,425 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76be007d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/hadoop.log.dir/,AVAILABLE} 2024-12-01T23:28:11,426 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75279722{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T23:28:11,573 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2df1c2c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/java.io.tmpdir/jetty-localhost-43529-hadoop-hdfs-3_4_1-tests_jar-_-any-5346373559538372454/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T23:28:11,573 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1cfa1d{HTTP/1.1, (http/1.1)}{localhost:43529} 2024-12-01T23:28:11,574 INFO [Time-limited test {}] server.Server(415): Started @6650ms 2024-12-01T23:28:11,576 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T23:28:13,076 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data1/current/BP-959958609-172.17.0.3-1733095688324/current, will proceed with Du for space computation calculation, 2024-12-01T23:28:13,076 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data2/current/BP-959958609-172.17.0.3-1733095688324/current, will proceed with Du for space computation calculation, 2024-12-01T23:28:13,078 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data4/current/BP-959958609-172.17.0.3-1733095688324/current, will proceed with Du for space computation calculation, 2024-12-01T23:28:13,078 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data3/current/BP-959958609-172.17.0.3-1733095688324/current, will proceed with Du for space computation calculation, 2024-12-01T23:28:13,087 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data5/current/BP-959958609-172.17.0.3-1733095688324/current, will proceed with Du for space computation calculation, 2024-12-01T23:28:13,088 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data6/current/BP-959958609-172.17.0.3-1733095688324/current, will proceed with Du for space computation calculation, 2024-12-01T23:28:13,186 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T23:28:13,195 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T23:28:13,201 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T23:28:13,248 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x835d9d72cdcd5e34 with lease ID 0xacd9732ae4cd81ef: Processing first storage report for DS-0b73283e-be70-47e5-a2bb-bfd83dc14319 from datanode DatanodeRegistration(127.0.0.1:41911, datanodeUuid=600d60c5-46af-4ad8-97d9-3329e790967e, infoPort=37389, infoSecurePort=0, ipcPort=42131, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324) 2024-12-01T23:28:13,250 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x835d9d72cdcd5e34 with lease ID 0xacd9732ae4cd81ef: from storage DS-0b73283e-be70-47e5-a2bb-bfd83dc14319 node DatanodeRegistration(127.0.0.1:41911, datanodeUuid=600d60c5-46af-4ad8-97d9-3329e790967e, infoPort=37389, infoSecurePort=0, ipcPort=42131, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-01T23:28:13,250 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xac26661aade7ab85 with lease ID 0xacd9732ae4cd81ed: Processing first storage report for DS-23f7379b-d6a8-4394-9494-ce51b3d3d379 from datanode DatanodeRegistration(127.0.0.1:44369, datanodeUuid=1b761c6e-5c73-49cf-aa4f-58c634bd83b5, infoPort=44317, infoSecurePort=0, ipcPort=46003, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324) 2024-12-01T23:28:13,251 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xac26661aade7ab85 with lease ID 0xacd9732ae4cd81ed: from storage DS-23f7379b-d6a8-4394-9494-ce51b3d3d379 node DatanodeRegistration(127.0.0.1:44369, datanodeUuid=1b761c6e-5c73-49cf-aa4f-58c634bd83b5, infoPort=44317, infoSecurePort=0, ipcPort=46003, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T23:28:13,251 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdb7df221d11bdd3d with lease ID 0xacd9732ae4cd81ee: Processing first storage report for DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c from datanode DatanodeRegistration(127.0.0.1:39219, datanodeUuid=4b04c2ae-3180-4679-9524-d641def164bb, infoPort=36067, infoSecurePort=0, ipcPort=42391, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324) 2024-12-01T23:28:13,251 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdb7df221d11bdd3d with lease ID 0xacd9732ae4cd81ee: from storage DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c node DatanodeRegistration(127.0.0.1:39219, datanodeUuid=4b04c2ae-3180-4679-9524-d641def164bb, infoPort=36067, infoSecurePort=0, ipcPort=42391, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-01T23:28:13,251 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x835d9d72cdcd5e34 with lease ID 0xacd9732ae4cd81ef: Processing first storage report for DS-fe4495f1-3e41-4f54-a724-34ed8656b4c4 from datanode DatanodeRegistration(127.0.0.1:41911, datanodeUuid=600d60c5-46af-4ad8-97d9-3329e790967e, infoPort=37389, infoSecurePort=0, ipcPort=42131, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324) 2024-12-01T23:28:13,252 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x835d9d72cdcd5e34 with lease ID 0xacd9732ae4cd81ef: from storage DS-fe4495f1-3e41-4f54-a724-34ed8656b4c4 node DatanodeRegistration(127.0.0.1:41911, datanodeUuid=600d60c5-46af-4ad8-97d9-3329e790967e, infoPort=37389, infoSecurePort=0, ipcPort=42131, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T23:28:13,252 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xac26661aade7ab85 with lease ID 0xacd9732ae4cd81ed: Processing first storage report for DS-830bc4b2-b977-4f46-81a6-3d7e9f10dcee from datanode DatanodeRegistration(127.0.0.1:44369, datanodeUuid=1b761c6e-5c73-49cf-aa4f-58c634bd83b5, infoPort=44317, infoSecurePort=0, ipcPort=46003, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324) 2024-12-01T23:28:13,252 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xac26661aade7ab85 with lease ID 0xacd9732ae4cd81ed: from storage DS-830bc4b2-b977-4f46-81a6-3d7e9f10dcee node DatanodeRegistration(127.0.0.1:44369, datanodeUuid=1b761c6e-5c73-49cf-aa4f-58c634bd83b5, infoPort=44317, infoSecurePort=0, ipcPort=46003, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-01T23:28:13,252 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdb7df221d11bdd3d with lease ID 0xacd9732ae4cd81ee: Processing first storage report for DS-2fe897e1-91df-4f10-a800-00046cd7186b from datanode DatanodeRegistration(127.0.0.1:39219, datanodeUuid=4b04c2ae-3180-4679-9524-d641def164bb, infoPort=36067, infoSecurePort=0, ipcPort=42391, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324) 2024-12-01T23:28:13,253 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdb7df221d11bdd3d with lease ID 0xacd9732ae4cd81ee: from storage DS-2fe897e1-91df-4f10-a800-00046cd7186b node DatanodeRegistration(127.0.0.1:39219, datanodeUuid=4b04c2ae-3180-4679-9524-d641def164bb, infoPort=36067, infoSecurePort=0, ipcPort=42391, storageInfo=lv=-57;cid=testClusterID;nsid=1388342852;c=1733095688324), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T23:28:13,314 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72 2024-12-01T23:28:13,430 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/zookeeper_0, clientPort=56655, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-01T23:28:13,446 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56655 2024-12-01T23:28:13,462 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:13,466 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:13,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741825_1001 (size=7) 2024-12-01T23:28:13,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741825_1001 (size=7) 2024-12-01T23:28:13,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741825_1001 (size=7) 2024-12-01T23:28:14,288 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0 with version=8 2024-12-01T23:28:14,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/hbase-staging 2024-12-01T23:28:14,655 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f962de17fdc3:0 server-side Connection retries=45 2024-12-01T23:28:14,664 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:14,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:14,670 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:28:14,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:14,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:28:14,828 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-01T23:28:14,897 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-01T23:28:14,909 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-01T23:28:14,913 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:28:14,946 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 58157 (auto-detected) 2024-12-01T23:28:14,947 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-01T23:28:14,972 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36351 2024-12-01T23:28:14,997 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36351 connecting to ZooKeeper ensemble=127.0.0.1:56655 2024-12-01T23:28:15,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:363510x0, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:28:15,154 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36351-0x101933997fc0000 connected 2024-12-01T23:28:15,246 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:15,251 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:15,264 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:28:15,268 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0, hbase.cluster.distributed=false 2024-12-01T23:28:15,298 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:28:15,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36351 2024-12-01T23:28:15,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36351 2024-12-01T23:28:15,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36351 2024-12-01T23:28:15,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36351 2024-12-01T23:28:15,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36351 2024-12-01T23:28:15,420 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f962de17fdc3:0 server-side Connection retries=45 2024-12-01T23:28:15,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:15,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:15,422 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:28:15,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:15,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:28:15,425 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T23:28:15,428 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:28:15,429 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34145 2024-12-01T23:28:15,430 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34145 connecting to ZooKeeper ensemble=127.0.0.1:56655 2024-12-01T23:28:15,432 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:15,436 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:15,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341450x0, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:28:15,477 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:341450x0, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:28:15,477 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34145-0x101933997fc0001 connected 2024-12-01T23:28:15,482 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T23:28:15,492 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T23:28:15,496 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T23:28:15,501 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:28:15,506 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34145 2024-12-01T23:28:15,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34145 2024-12-01T23:28:15,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34145 2024-12-01T23:28:15,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34145 2024-12-01T23:28:15,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34145 2024-12-01T23:28:15,538 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f962de17fdc3:0 server-side Connection retries=45 2024-12-01T23:28:15,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:15,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:15,539 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:28:15,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:15,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:28:15,540 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T23:28:15,540 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:28:15,543 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33945 2024-12-01T23:28:15,546 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33945 connecting to ZooKeeper ensemble=127.0.0.1:56655 2024-12-01T23:28:15,547 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:15,552 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:15,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:339450x0, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:28:15,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:339450x0, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:28:15,571 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33945-0x101933997fc0002 connected 2024-12-01T23:28:15,572 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T23:28:15,573 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T23:28:15,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T23:28:15,576 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:28:15,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33945 2024-12-01T23:28:15,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33945 2024-12-01T23:28:15,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33945 2024-12-01T23:28:15,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33945 2024-12-01T23:28:15,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33945 2024-12-01T23:28:15,604 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f962de17fdc3:0 server-side Connection retries=45 2024-12-01T23:28:15,604 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:15,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:15,605 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T23:28:15,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T23:28:15,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T23:28:15,605 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T23:28:15,606 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T23:28:15,610 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39661 2024-12-01T23:28:15,613 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39661 connecting to ZooKeeper ensemble=127.0.0.1:56655 2024-12-01T23:28:15,617 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:15,621 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:15,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396610x0, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T23:28:15,651 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39661-0x101933997fc0003 connected 2024-12-01T23:28:15,653 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:28:15,657 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T23:28:15,664 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T23:28:15,676 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T23:28:15,686 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T23:28:15,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39661 2024-12-01T23:28:15,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39661 2024-12-01T23:28:15,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39661 2024-12-01T23:28:15,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39661 2024-12-01T23:28:15,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39661 2024-12-01T23:28:15,713 DEBUG [M:0;f962de17fdc3:36351 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f962de17fdc3:36351 2024-12-01T23:28:15,715 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f962de17fdc3,36351,1733095694462 2024-12-01T23:28:15,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:28:15,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:28:15,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:28:15,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:28:15,744 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f962de17fdc3,36351,1733095694462 2024-12-01T23:28:15,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T23:28:15,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T23:28:15,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:15,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:15,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:15,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T23:28:15,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:15,798 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-01T23:28:15,801 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f962de17fdc3,36351,1733095694462 from backup master directory 2024-12-01T23:28:15,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:28:15,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f962de17fdc3,36351,1733095694462 2024-12-01T23:28:15,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:28:15,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:28:15,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T23:28:15,883 WARN [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:28:15,883 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f962de17fdc3,36351,1733095694462 2024-12-01T23:28:15,886 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-01T23:28:15,890 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-01T23:28:15,979 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/hbase.id] with ID: 3b9d21d3-1a5d-4ed1-a1a3-dd57abfb96d0 2024-12-01T23:28:15,980 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/.tmp/hbase.id 2024-12-01T23:28:16,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741826_1002 (size=42) 2024-12-01T23:28:16,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741826_1002 (size=42) 2024-12-01T23:28:16,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741826_1002 (size=42) 2024-12-01T23:28:16,414 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/.tmp/hbase.id]:[hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/hbase.id] 2024-12-01T23:28:16,512 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T23:28:16,522 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-01T23:28:16,551 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-12-01T23:28:16,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:16,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:16,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:16,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:17,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741827_1003 (size=196) 2024-12-01T23:28:17,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741827_1003 (size=196) 2024-12-01T23:28:17,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741827_1003 (size=196) 2024-12-01T23:28:17,348 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T23:28:17,351 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-01T23:28:17,360 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:17,417 WARN [IPC Server handler 0 on default port 35981 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-01T23:28:17,418 WARN [IPC Server handler 0 on default port 35981 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-01T23:28:17,418 WARN [IPC Server handler 0 on default port 35981 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-01T23:28:17,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741828_1004 (size=1189) 2024-12-01T23:28:17,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741828_1004 (size=1189) 2024-12-01T23:28:17,485 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store 2024-12-01T23:28:17,519 WARN [IPC Server handler 3 on default port 35981 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-01T23:28:17,520 WARN [IPC Server handler 3 on default port 35981 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-01T23:28:17,520 WARN [IPC Server handler 3 on default port 35981 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-01T23:28:17,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741829_1005 (size=34) 2024-12-01T23:28:17,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741829_1005 (size=34) 2024-12-01T23:28:17,581 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-01T23:28:17,585 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:17,587 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T23:28:17,587 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:28:17,588 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:28:17,590 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T23:28:17,591 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:28:17,591 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:28:17,593 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733095697587Disabling compacts and flushes for region at 1733095697587Disabling writes for close at 1733095697591 (+4 ms)Writing region close event to WAL at 1733095697591Closed at 1733095697591 2024-12-01T23:28:17,597 WARN [master/f962de17fdc3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/.initializing 2024-12-01T23:28:17,598 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/WALs/f962de17fdc3,36351,1733095694462 2024-12-01T23:28:17,616 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:17,645 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f962de17fdc3%2C36351%2C1733095694462, suffix=, logDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/WALs/f962de17fdc3,36351,1733095694462, archiveDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/oldWALs, maxLogs=10 2024-12-01T23:28:17,694 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/WALs/f962de17fdc3,36351,1733095694462/f962de17fdc3%2C36351%2C1733095694462.1733095697652, exclude list is [], retry=0 2024-12-01T23:28:17,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:17,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:17,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:17,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:17,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-01T23:28:17,782 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-01T23:28:18,065 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/WALs/f962de17fdc3,36351,1733095694462/f962de17fdc3%2C36351%2C1733095694462.1733095697652 2024-12-01T23:28:18,070 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:18,071 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:18,072 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:18,077 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,078 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-01T23:28:18,191 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:18,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:18,197 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,206 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-01T23:28:18,206 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:18,208 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:18,208 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-01T23:28:18,215 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:18,217 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:18,220 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,225 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-01T23:28:18,225 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:18,227 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:18,228 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,239 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,241 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,254 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,255 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,260 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:28:18,267 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T23:28:18,275 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:18,279 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68503524, jitterRate=0.020782053470611572}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:28:18,289 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733095698100Initializing all the Stores at 1733095698104 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733095698105 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095698106 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095698107 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095698107Cleaning up temporary data from old regions at 1733095698255 (+148 ms)Region opened successfully at 1733095698289 (+34 ms) 2024-12-01T23:28:18,291 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-01T23:28:18,354 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1443fe88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f962de17fdc3/172.17.0.3:0 2024-12-01T23:28:18,403 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-01T23:28:18,422 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-01T23:28:18,422 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-01T23:28:18,427 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-01T23:28:18,431 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-12-01T23:28:18,439 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-01T23:28:18,440 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-01T23:28:18,482 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-01T23:28:18,496 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-01T23:28:18,554 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-01T23:28:18,558 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-01T23:28:18,561 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-01T23:28:18,570 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-01T23:28:18,573 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-01T23:28:18,579 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-01T23:28:18,591 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-01T23:28:18,593 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-01T23:28:18,607 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-01T23:28:18,631 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-01T23:28:18,731 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-01T23:28:18,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T23:28:18,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T23:28:18,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T23:28:18,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T23:28:18,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,811 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f962de17fdc3,36351,1733095694462, sessionid=0x101933997fc0000, setting cluster-up flag (Was=false) 2024-12-01T23:28:18,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,965 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-01T23:28:18,967 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f962de17fdc3,36351,1733095694462 2024-12-01T23:28:18,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:18,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:19,022 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-01T23:28:19,025 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f962de17fdc3,36351,1733095694462 2024-12-01T23:28:19,036 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-01T23:28:19,127 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(746): ClusterId : 3b9d21d3-1a5d-4ed1-a1a3-dd57abfb96d0 2024-12-01T23:28:19,130 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T23:28:19,133 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(746): ClusterId : 3b9d21d3-1a5d-4ed1-a1a3-dd57abfb96d0 2024-12-01T23:28:19,133 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T23:28:19,134 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(746): ClusterId : 3b9d21d3-1a5d-4ed1-a1a3-dd57abfb96d0 2024-12-01T23:28:19,135 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T23:28:19,162 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T23:28:19,163 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T23:28:19,166 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T23:28:19,167 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T23:28:19,189 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T23:28:19,189 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T23:28:19,189 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T23:28:19,190 DEBUG [RS:1;f962de17fdc3:33945 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f1e7f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f962de17fdc3/172.17.0.3:0 2024-12-01T23:28:19,187 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-01T23:28:19,213 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T23:28:19,214 DEBUG [RS:0;f962de17fdc3:34145 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7793ca27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f962de17fdc3/172.17.0.3:0 2024-12-01T23:28:19,215 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-01T23:28:19,235 DEBUG [RS:1;f962de17fdc3:33945 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f962de17fdc3:33945 2024-12-01T23:28:19,241 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T23:28:19,242 DEBUG [RS:2;f962de17fdc3:39661 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57055c02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f962de17fdc3/172.17.0.3:0 2024-12-01T23:28:19,247 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f962de17fdc3:34145 2024-12-01T23:28:19,249 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T23:28:19,250 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T23:28:19,250 DEBUG [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T23:28:19,254 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-01T23:28:19,258 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(2659): reportForDuty to master=f962de17fdc3,36351,1733095694462 with port=33945, startcode=1733095695537 2024-12-01T23:28:19,259 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T23:28:19,259 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T23:28:19,259 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T23:28:19,265 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(2659): reportForDuty to master=f962de17fdc3,36351,1733095694462 with port=34145, startcode=1733095695383 2024-12-01T23:28:19,277 DEBUG [RS:2;f962de17fdc3:39661 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f962de17fdc3:39661 2024-12-01T23:28:19,278 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T23:28:19,278 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T23:28:19,285 DEBUG [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T23:28:19,276 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f962de17fdc3,36351,1733095694462 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-01T23:28:19,290 DEBUG [RS:1;f962de17fdc3:33945 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T23:28:19,291 DEBUG [RS:0;f962de17fdc3:34145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T23:28:19,292 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(2659): reportForDuty to master=f962de17fdc3,36351,1733095694462 with port=39661, startcode=1733095695603 2024-12-01T23:28:19,293 DEBUG [RS:2;f962de17fdc3:39661 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T23:28:19,321 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f962de17fdc3:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:28:19,322 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f962de17fdc3:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:28:19,322 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f962de17fdc3:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:28:19,322 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f962de17fdc3:0, corePoolSize=5, maxPoolSize=5 2024-12-01T23:28:19,322 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f962de17fdc3:0, corePoolSize=10, maxPoolSize=10 2024-12-01T23:28:19,322 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:19,322 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f962de17fdc3:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:28:19,323 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:19,367 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733095729367 2024-12-01T23:28:19,389 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50037, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T23:28:19,390 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59403, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T23:28:19,390 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47141, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T23:28:19,390 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-01T23:28:19,392 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-01T23:28:19,396 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-01T23:28:19,397 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-01T23:28:19,397 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T23:28:19,398 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-01T23:28:19,398 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-01T23:28:19,399 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-01T23:28:19,399 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-01T23:28:19,400 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:19,417 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-01T23:28:19,421 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-01T23:28:19,422 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-01T23:28:19,423 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-01T23:28:19,423 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:19,424 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-01T23:28:19,426 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-01T23:28:19,426 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-01T23:28:19,427 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-01T23:28:19,433 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f962de17fdc3:0:becomeActiveMaster-HFileCleaner.large.0-1733095699428,5,FailOnTimeoutGroup] 2024-12-01T23:28:19,442 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f962de17fdc3:0:becomeActiveMaster-HFileCleaner.small.0-1733095699434,5,FailOnTimeoutGroup] 2024-12-01T23:28:19,442 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:19,442 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-01T23:28:19,445 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:19,446 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:19,531 DEBUG [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-01T23:28:19,532 WARN [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-01T23:28:19,535 DEBUG [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-01T23:28:19,535 WARN [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-01T23:28:19,535 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-01T23:28:19,535 WARN [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-01T23:28:19,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741831_1007 (size=1321) 2024-12-01T23:28:19,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741831_1007 (size=1321) 2024-12-01T23:28:19,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741831_1007 (size=1321) 2024-12-01T23:28:19,600 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-01T23:28:19,601 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0 2024-12-01T23:28:19,640 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(2659): reportForDuty to master=f962de17fdc3,36351,1733095694462 with port=34145, startcode=1733095695383 2024-12-01T23:28:19,643 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(2659): reportForDuty to master=f962de17fdc3,36351,1733095694462 with port=33945, startcode=1733095695537 2024-12-01T23:28:19,643 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f962de17fdc3,34145,1733095695383 2024-12-01T23:28:19,647 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] master.ServerManager(517): Registering regionserver=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:19,647 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(2659): reportForDuty to master=f962de17fdc3,36351,1733095694462 with port=39661, startcode=1733095695603 2024-12-01T23:28:19,683 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f962de17fdc3,33945,1733095695537 2024-12-01T23:28:19,687 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] master.ServerManager(517): Registering regionserver=f962de17fdc3,33945,1733095695537 2024-12-01T23:28:19,692 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0 2024-12-01T23:28:19,692 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35981 2024-12-01T23:28:19,692 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T23:28:19,702 DEBUG [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0 2024-12-01T23:28:19,702 DEBUG [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35981 2024-12-01T23:28:19,702 DEBUG [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T23:28:19,703 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f962de17fdc3,39661,1733095695603 2024-12-01T23:28:19,703 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] master.ServerManager(517): Registering regionserver=f962de17fdc3,39661,1733095695603 2024-12-01T23:28:19,711 DEBUG [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0 2024-12-01T23:28:19,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741832_1008 (size=32) 2024-12-01T23:28:19,711 DEBUG [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35981 2024-12-01T23:28:19,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741832_1008 (size=32) 2024-12-01T23:28:19,711 DEBUG [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T23:28:19,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741832_1008 (size=32) 2024-12-01T23:28:19,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:19,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T23:28:19,738 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T23:28:19,738 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:19,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:19,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T23:28:19,762 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T23:28:19,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:19,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:19,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T23:28:19,794 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T23:28:19,795 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:19,800 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:19,800 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T23:28:19,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T23:28:19,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:19,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:19,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T23:28:19,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740 2024-12-01T23:28:19,829 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740 2024-12-01T23:28:19,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T23:28:19,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T23:28:19,851 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:28:19,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T23:28:19,888 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:19,893 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74480023, jitterRate=0.1098388284444809}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:28:19,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733095699715Initializing all the Stores at 1733095699721 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733095699722 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733095699724 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095699724Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733095699724Cleaning up temporary data from old regions at 1733095699848 (+124 ms)Region opened successfully at 1733095699897 (+49 ms) 2024-12-01T23:28:19,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T23:28:19,898 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T23:28:19,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T23:28:19,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T23:28:19,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T23:28:19,908 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T23:28:19,908 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733095699898Disabling compacts and flushes for region at 1733095699898Disabling writes for close at 1733095699899 (+1 ms)Writing region close event to WAL at 1733095699907 (+8 ms)Closed at 1733095699908 (+1 ms) 2024-12-01T23:28:19,925 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T23:28:19,925 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-01T23:28:19,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T23:28:19,967 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T23:28:19,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:28:20,022 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-01T23:28:20,063 DEBUG [RS:0;f962de17fdc3:34145 {}] zookeeper.ZKUtil(111): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f962de17fdc3,34145,1733095695383 2024-12-01T23:28:20,063 WARN [RS:0;f962de17fdc3:34145 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:28:20,064 INFO [RS:0;f962de17fdc3:34145 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:20,064 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,34145,1733095695383 2024-12-01T23:28:20,065 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f962de17fdc3,33945,1733095695537] 2024-12-01T23:28:20,068 DEBUG [RS:1;f962de17fdc3:33945 {}] zookeeper.ZKUtil(111): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f962de17fdc3,33945,1733095695537 2024-12-01T23:28:20,068 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f962de17fdc3,34145,1733095695383] 2024-12-01T23:28:20,068 DEBUG [RS:2;f962de17fdc3:39661 {}] zookeeper.ZKUtil(111): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f962de17fdc3,39661,1733095695603 2024-12-01T23:28:20,068 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f962de17fdc3,39661,1733095695603] 2024-12-01T23:28:20,068 WARN [RS:2;f962de17fdc3:39661 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:28:20,068 WARN [RS:1;f962de17fdc3:33945 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T23:28:20,068 INFO [RS:1;f962de17fdc3:33945 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:20,068 INFO [RS:2;f962de17fdc3:39661 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:20,069 DEBUG [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603 2024-12-01T23:28:20,069 DEBUG [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,33945,1733095695537 2024-12-01T23:28:20,130 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T23:28:20,152 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T23:28:20,163 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T23:28:20,168 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T23:28:20,168 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T23:28:20,174 WARN [f962de17fdc3:36351 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T23:28:20,179 INFO [RS:2;f962de17fdc3:39661 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T23:28:20,179 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,180 INFO [RS:0;f962de17fdc3:34145 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T23:28:20,180 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,186 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T23:28:20,194 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T23:28:20,196 INFO [RS:1;f962de17fdc3:33945 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T23:28:20,196 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,197 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T23:28:20,199 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,199 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,200 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,200 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T23:28:20,201 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,202 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,202 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,202 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:28:20,202 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,202 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,203 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,203 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,203 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,203 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,203 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f962de17fdc3:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:28:20,203 DEBUG [RS:2;f962de17fdc3:39661 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f962de17fdc3:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:28:20,204 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T23:28:20,210 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,211 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,211 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,215 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T23:28:20,216 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,220 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,220 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,221 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T23:28:20,221 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,221 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,221 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,221 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,221 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,221 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:28:20,221 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,221 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,221 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0, corePoolSize=2, maxPoolSize=2 2024-12-01T23:28:20,222 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f962de17fdc3:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:28:20,222 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,222 DEBUG [RS:0;f962de17fdc3:34145 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f962de17fdc3:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:28:20,222 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,223 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,223 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f962de17fdc3:0, corePoolSize=1, maxPoolSize=1 2024-12-01T23:28:20,223 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f962de17fdc3:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:28:20,223 DEBUG [RS:1;f962de17fdc3:33945 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f962de17fdc3:0, corePoolSize=3, maxPoolSize=3 2024-12-01T23:28:20,225 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,225 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,226 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,39661,1733095695603-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:28:20,242 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,242 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,242 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,243 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,243 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,243 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,33945,1733095695537-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:28:20,284 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,284 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,285 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,285 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,285 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,285 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,34145,1733095695383-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:28:20,296 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T23:28:20,302 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T23:28:20,307 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,33945,1733095695537-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,308 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,308 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,39661,1733095695603-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,309 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,309 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.Replication(171): f962de17fdc3,39661,1733095695603 started 2024-12-01T23:28:20,309 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.Replication(171): f962de17fdc3,33945,1733095695537 started 2024-12-01T23:28:20,316 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T23:28:20,316 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,34145,1733095695383-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,317 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,317 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.Replication(171): f962de17fdc3,34145,1733095695383 started 2024-12-01T23:28:20,343 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,345 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(1482): Serving as f962de17fdc3,33945,1733095695537, RpcServer on f962de17fdc3/172.17.0.3:33945, sessionid=0x101933997fc0002 2024-12-01T23:28:20,346 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T23:28:20,346 DEBUG [RS:1;f962de17fdc3:33945 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f962de17fdc3,33945,1733095695537 2024-12-01T23:28:20,347 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f962de17fdc3,33945,1733095695537' 2024-12-01T23:28:20,347 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T23:28:20,348 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T23:28:20,349 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T23:28:20,349 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T23:28:20,350 DEBUG [RS:1;f962de17fdc3:33945 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f962de17fdc3,33945,1733095695537 2024-12-01T23:28:20,350 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f962de17fdc3,33945,1733095695537' 2024-12-01T23:28:20,350 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T23:28:20,351 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T23:28:20,352 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,356 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1482): Serving as f962de17fdc3,34145,1733095695383, RpcServer on f962de17fdc3/172.17.0.3:34145, sessionid=0x101933997fc0001 2024-12-01T23:28:20,357 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:20,357 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T23:28:20,357 DEBUG [RS:0;f962de17fdc3:34145 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f962de17fdc3,34145,1733095695383 2024-12-01T23:28:20,352 DEBUG [RS:1;f962de17fdc3:33945 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T23:28:20,357 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f962de17fdc3,34145,1733095695383' 2024-12-01T23:28:20,357 INFO [RS:1;f962de17fdc3:33945 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T23:28:20,357 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T23:28:20,357 INFO [RS:1;f962de17fdc3:33945 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T23:28:20,358 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(1482): Serving as f962de17fdc3,39661,1733095695603, RpcServer on f962de17fdc3/172.17.0.3:39661, sessionid=0x101933997fc0003 2024-12-01T23:28:20,359 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T23:28:20,359 DEBUG [RS:2;f962de17fdc3:39661 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f962de17fdc3,39661,1733095695603 2024-12-01T23:28:20,359 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f962de17fdc3,39661,1733095695603' 2024-12-01T23:28:20,359 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T23:28:20,369 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T23:28:20,372 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T23:28:20,382 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T23:28:20,382 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T23:28:20,382 DEBUG [RS:0;f962de17fdc3:34145 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f962de17fdc3,34145,1733095695383 2024-12-01T23:28:20,383 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f962de17fdc3,34145,1733095695383' 2024-12-01T23:28:20,383 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T23:28:20,384 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T23:28:20,384 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T23:28:20,385 DEBUG [RS:2;f962de17fdc3:39661 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f962de17fdc3,39661,1733095695603 2024-12-01T23:28:20,385 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f962de17fdc3,39661,1733095695603' 2024-12-01T23:28:20,385 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T23:28:20,390 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T23:28:20,392 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T23:28:20,401 DEBUG [RS:2;f962de17fdc3:39661 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T23:28:20,401 INFO [RS:2;f962de17fdc3:39661 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T23:28:20,401 INFO [RS:2;f962de17fdc3:39661 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T23:28:20,401 DEBUG [RS:0;f962de17fdc3:34145 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T23:28:20,401 INFO [RS:0;f962de17fdc3:34145 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T23:28:20,401 INFO [RS:0;f962de17fdc3:34145 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T23:28:20,464 INFO [RS:1;f962de17fdc3:33945 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:20,470 INFO [RS:1;f962de17fdc3:33945 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f962de17fdc3%2C33945%2C1733095695537, suffix=, logDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,33945,1733095695537, archiveDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/oldWALs, maxLogs=32 2024-12-01T23:28:20,493 DEBUG [RS:1;f962de17fdc3:33945 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,33945,1733095695537/f962de17fdc3%2C33945%2C1733095695537.1733095700474, exclude list is [], retry=0 2024-12-01T23:28:20,509 INFO [RS:2;f962de17fdc3:39661 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:20,511 INFO [RS:0;f962de17fdc3:34145 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:20,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:20,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:20,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:20,520 INFO [RS:2;f962de17fdc3:39661 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f962de17fdc3%2C39661%2C1733095695603, suffix=, logDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603, archiveDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/oldWALs, maxLogs=32 2024-12-01T23:28:20,523 INFO [RS:0;f962de17fdc3:34145 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f962de17fdc3%2C34145%2C1733095695383, suffix=, logDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,34145,1733095695383, archiveDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/oldWALs, maxLogs=32 2024-12-01T23:28:20,542 DEBUG [RS:2;f962de17fdc3:39661 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603/f962de17fdc3%2C39661%2C1733095695603.1733095700522, exclude list is [], retry=0 2024-12-01T23:28:20,568 DEBUG [RS:0;f962de17fdc3:34145 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,34145,1733095695383/f962de17fdc3%2C34145%2C1733095695383.1733095700525, exclude list is [], retry=0 2024-12-01T23:28:20,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:20,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:20,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:20,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:20,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:20,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:20,644 INFO [RS:1;f962de17fdc3:33945 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,33945,1733095695537/f962de17fdc3%2C33945%2C1733095695537.1733095700474 2024-12-01T23:28:20,652 DEBUG [RS:1;f962de17fdc3:33945 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:20,676 INFO [RS:2;f962de17fdc3:39661 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603/f962de17fdc3%2C39661%2C1733095695603.1733095700522 2024-12-01T23:28:20,678 DEBUG [RS:2;f962de17fdc3:39661 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:28:20,722 INFO [RS:0;f962de17fdc3:34145 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,34145,1733095695383/f962de17fdc3%2C34145%2C1733095695383.1733095700525 2024-12-01T23:28:20,726 DEBUG [RS:0;f962de17fdc3:34145 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:28:20,928 DEBUG [f962de17fdc3:36351 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-01T23:28:20,939 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(204): Hosts are {f962de17fdc3=0} racks are {/default-rack=0} 2024-12-01T23:28:20,961 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T23:28:20,961 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T23:28:20,962 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-01T23:28:20,962 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T23:28:20,962 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T23:28:20,962 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-01T23:28:20,962 INFO [f962de17fdc3:36351 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T23:28:20,962 INFO [f962de17fdc3:36351 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T23:28:20,962 INFO [f962de17fdc3:36351 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-01T23:28:20,962 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T23:28:20,979 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f962de17fdc3,39661,1733095695603 2024-12-01T23:28:20,986 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f962de17fdc3,39661,1733095695603, state=OPENING 2024-12-01T23:28:21,048 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-01T23:28:21,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:21,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:21,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:21,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:28:21,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:21,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:21,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:21,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:21,063 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T23:28:21,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f962de17fdc3,39661,1733095695603}] 2024-12-01T23:28:21,260 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T23:28:21,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42997, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T23:28:21,308 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T23:28:21,309 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:21,310 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T23:28:21,330 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f962de17fdc3%2C39661%2C1733095695603.meta, suffix=.meta, logDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603, archiveDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/oldWALs, maxLogs=32 2024-12-01T23:28:21,360 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta, exclude list is [], retry=0 2024-12-01T23:28:21,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:21,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:21,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:21,478 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta 2024-12-01T23:28:21,483 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:28:21,483 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:21,486 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T23:28:21,490 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T23:28:21,497 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T23:28:21,503 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T23:28:21,504 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:21,505 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T23:28:21,505 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T23:28:21,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T23:28:21,519 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T23:28:21,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:21,520 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:21,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T23:28:21,523 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T23:28:21,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:21,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:21,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T23:28:21,531 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T23:28:21,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:21,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:21,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T23:28:21,535 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T23:28:21,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:21,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:21,537 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T23:28:21,539 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740 2024-12-01T23:28:21,556 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740 2024-12-01T23:28:21,563 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T23:28:21,563 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T23:28:21,565 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:28:21,569 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T23:28:21,574 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66784866, jitterRate=-0.004827946424484253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:28:21,575 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T23:28:21,576 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733095701505Writing region info on filesystem at 1733095701506 (+1 ms)Initializing all the Stores at 1733095701513 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733095701513Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733095701516 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095701516Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733095701516Cleaning up temporary data from old regions at 1733095701563 (+47 ms)Running coprocessor post-open hooks at 1733095701575 (+12 ms)Region opened successfully at 1733095701576 (+1 ms) 2024-12-01T23:28:21,596 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733095701247 2024-12-01T23:28:21,625 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T23:28:21,626 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T23:28:21,630 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f962de17fdc3,39661,1733095695603 2024-12-01T23:28:21,633 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f962de17fdc3,39661,1733095695603, state=OPEN 2024-12-01T23:28:21,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:28:21,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:28:21,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:28:21,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:28:21,680 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:21,680 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:21,680 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:21,681 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:21,681 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f962de17fdc3,39661,1733095695603 2024-12-01T23:28:21,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-01T23:28:21,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f962de17fdc3,39661,1733095695603 in 615 msec 2024-12-01T23:28:21,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-01T23:28:21,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.7530 sec 2024-12-01T23:28:21,728 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T23:28:21,729 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-01T23:28:21,766 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:28:21,768 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1] 2024-12-01T23:28:21,794 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T23:28:21,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50341, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T23:28:21,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 2.7550 sec 2024-12-01T23:28:21,859 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733095701858, completionTime=-1 2024-12-01T23:28:21,862 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-01T23:28:21,862 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-01T23:28:21,914 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-01T23:28:21,914 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733095761914 2024-12-01T23:28:21,914 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733095821914 2024-12-01T23:28:21,914 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 51 msec 2024-12-01T23:28:21,917 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-01T23:28:21,945 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,36351,1733095694462-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:21,946 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,36351,1733095694462-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:21,946 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,36351,1733095694462-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:21,948 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f962de17fdc3:36351, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:21,949 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:21,951 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:21,973 DEBUG [master/f962de17fdc3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-01T23:28:21,999 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 6.116sec 2024-12-01T23:28:22,013 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-01T23:28:22,015 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-01T23:28:22,016 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-01T23:28:22,017 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-01T23:28:22,017 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-01T23:28:22,018 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,36351,1733095694462-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T23:28:22,019 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,36351,1733095694462-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-01T23:28:22,029 DEBUG [master/f962de17fdc3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-01T23:28:22,031 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-01T23:28:22,031 INFO [master/f962de17fdc3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f962de17fdc3,36351,1733095694462-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:22,059 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bcc2fcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T23:28:22,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f962de17fdc3,36351,-1 for getting cluster id 2024-12-01T23:28:22,074 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T23:28:22,093 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3b9d21d3-1a5d-4ed1-a1a3-dd57abfb96d0' 2024-12-01T23:28:22,104 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T23:28:22,105 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3b9d21d3-1a5d-4ed1-a1a3-dd57abfb96d0" 2024-12-01T23:28:22,105 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33356e70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T23:28:22,106 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f962de17fdc3,36351,-1] 2024-12-01T23:28:22,109 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T23:28:22,114 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:28:22,121 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T23:28:22,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60a37f9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T23:28:22,130 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:28:22,147 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1] 2024-12-01T23:28:22,148 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T23:28:22,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T23:28:22,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f962de17fdc3,36351,1733095694462 2024-12-01T23:28:22,213 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:35981/hbase 2024-12-01T23:28:22,240 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=360, OpenFileDescriptor=605, MaxFileDescriptor=1048576, SystemLoadAverage=1239, ProcessCount=11, AvailableMemoryMB=3650 2024-12-01T23:28:22,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741828_1004 (size=1189) 2024-12-01T23:28:22,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741829_1005 (size=34) 2024-12-01T23:28:22,273 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:22,279 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:22,281 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:22,300 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-85522467, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/hregion-85522467, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:22,326 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-85522467/hregion-85522467.1733095702303, exclude list is [], retry=0 2024-12-01T23:28:22,341 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:22,342 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:22,383 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:22,461 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-85522467/hregion-85522467.1733095702303 2024-12-01T23:28:22,467 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:28:22,468 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d169d286ae05bbc58fe1bae2c16c5b1f, NAME => 'testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/hbase 2024-12-01T23:28:22,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741838_1014 (size=64) 2024-12-01T23:28:22,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741838_1014 (size=64) 2024-12-01T23:28:22,614 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:22,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741838_1014 (size=64) 2024-12-01T23:28:22,624 INFO [StoreOpener-d169d286ae05bbc58fe1bae2c16c5b1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:22,637 INFO [StoreOpener-d169d286ae05bbc58fe1bae2c16c5b1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d169d286ae05bbc58fe1bae2c16c5b1f columnFamilyName a 2024-12-01T23:28:22,637 DEBUG [StoreOpener-d169d286ae05bbc58fe1bae2c16c5b1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:22,639 INFO [StoreOpener-d169d286ae05bbc58fe1bae2c16c5b1f-1 {}] regionserver.HStore(327): Store=d169d286ae05bbc58fe1bae2c16c5b1f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:22,639 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:22,641 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:22,642 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:22,644 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:22,644 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:22,659 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:22,676 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:22,681 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d169d286ae05bbc58fe1bae2c16c5b1f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65856837, jitterRate=-0.0186566561460495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:28:22,683 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d169d286ae05bbc58fe1bae2c16c5b1f: Writing region info on filesystem at 1733095702614Initializing all the Stores at 1733095702619 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095702619Cleaning up temporary data from old regions at 1733095702644 (+25 ms)Region opened successfully at 1733095702683 (+39 ms) 2024-12-01T23:28:22,683 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d169d286ae05bbc58fe1bae2c16c5b1f, disabling compactions & flushes 2024-12-01T23:28:22,683 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f. 2024-12-01T23:28:22,684 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f. 2024-12-01T23:28:22,684 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f. after waiting 0 ms 2024-12-01T23:28:22,684 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f. 2024-12-01T23:28:22,688 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f. 2024-12-01T23:28:22,688 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d169d286ae05bbc58fe1bae2c16c5b1f: Waiting for close lock at 1733095702683Disabling compacts and flushes for region at 1733095702683Disabling writes for close at 1733095702684 (+1 ms)Writing region close event to WAL at 1733095702688 (+4 ms)Closed at 1733095702688 2024-12-01T23:28:22,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741837_1013 (size=175) 2024-12-01T23:28:22,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741837_1013 (size=175) 2024-12-01T23:28:22,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741837_1013 (size=175) 2024-12-01T23:28:22,756 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:22,757 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-85522467:(num 1733095702303) 2024-12-01T23:28:22,762 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T23:28:22,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741839_1015 (size=469) 2024-12-01T23:28:22,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741839_1015 (size=469) 2024-12-01T23:28:22,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741839_1015 (size=469) 2024-12-01T23:28:22,940 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T23:28:23,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741840_1016 (size=322) 2024-12-01T23:28:23,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741840_1016 (size=322) 2024-12-01T23:28:23,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741840_1016 (size=322) 2024-12-01T23:28:23,068 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-1, size=469 (469bytes) 2024-12-01T23:28:23,069 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-01T23:28:23,069 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-01T23:28:23,069 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-1 2024-12-01T23:28:23,079 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-1 after 4ms 2024-12-01T23:28:23,117 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:23,118 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-1 took 52ms 2024-12-01T23:28:23,124 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-1 so closing down 2024-12-01T23:28:23,125 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:23,129 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-01T23:28:23,160 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000001-wal-1.temp 2024-12-01T23:28:23,161 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:23,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741841_1017 (size=469) 2024-12-01T23:28:23,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741841_1017 (size=469) 2024-12-01T23:28:23,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741841_1017 (size=469) 2024-12-01T23:28:23,175 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-01T23:28:23,178 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000002 2024-12-01T23:28:23,183 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 58 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-01T23:28:23,183 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-1, journal: Splitting hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-1, size=469 (469bytes) at 1733095703068Finishing writing output for hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-1 so closing down at 1733095703125 (+57 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000001-wal-1.temp at 1733095703160 (+35 ms)3 split writer threads finished at 1733095703161 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733095703175 (+14 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000002 at 1733095703178 (+3 ms)Processed 2 edits across 1 Regions in 58 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733095703183 (+5 ms) 2024-12-01T23:28:23,204 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-2, size=322 (322bytes) 2024-12-01T23:28:23,205 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-2 2024-12-01T23:28:23,206 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-2 after 0ms 2024-12-01T23:28:23,246 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:23,247 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-2 took 43ms 2024-12-01T23:28:23,253 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-2 so closing down 2024-12-01T23:28:23,253 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:23,255 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-01T23:28:23,291 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000002-wal-2.temp 2024-12-01T23:28:23,292 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:23,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741842_1018 (size=322) 2024-12-01T23:28:23,384 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-01T23:28:23,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741842_1018 (size=322) 2024-12-01T23:28:23,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741842_1018 (size=322) 2024-12-01T23:28:23,449 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:23,452 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-01T23:28:23,455 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 205 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-01T23:28:23,455 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-2, journal: Splitting hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-2, size=322 (322bytes) at 1733095703204Finishing writing output for hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-2 so closing down at 1733095703253 (+49 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000002-wal-2.temp at 1733095703291 (+38 ms)3 split writer threads finished at 1733095703292 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733095703384 (+92 ms)Processed 1 edits across 1 Regions in 205 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733095703455 (+71 ms) 2024-12-01T23:28:23,455 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:23,458 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:23,472 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal.1733095703458, exclude list is [], retry=0 2024-12-01T23:28:23,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:23,479 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:23,479 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:23,526 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal.1733095703458 2024-12-01T23:28:23,527 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:23,527 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d169d286ae05bbc58fe1bae2c16c5b1f, NAME => 'testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:23,527 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:23,527 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:23,527 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:23,535 INFO [StoreOpener-d169d286ae05bbc58fe1bae2c16c5b1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:23,539 INFO [StoreOpener-d169d286ae05bbc58fe1bae2c16c5b1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d169d286ae05bbc58fe1bae2c16c5b1f columnFamilyName a 2024-12-01T23:28:23,539 DEBUG [StoreOpener-d169d286ae05bbc58fe1bae2c16c5b1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:23,540 INFO [StoreOpener-d169d286ae05bbc58fe1bae2c16c5b1f-1 {}] regionserver.HStore(327): Store=d169d286ae05bbc58fe1bae2c16c5b1f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:23,541 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:23,542 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:23,548 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:23,549 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000002 2024-12-01T23:28:23,576 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:23,581 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000002 2024-12-01T23:28:23,584 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d169d286ae05bbc58fe1bae2c16c5b1f 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-01T23:28:23,630 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/.tmp/a/eff14afa5cc4431298d457ec68b20977 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733095702757/Put/seqid=0 2024-12-01T23:28:23,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741844_1020 (size=5170) 2024-12-01T23:28:23,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741844_1020 (size=5170) 2024-12-01T23:28:23,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741844_1020 (size=5170) 2024-12-01T23:28:24,048 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/.tmp/a/eff14afa5cc4431298d457ec68b20977 2024-12-01T23:28:24,090 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/.tmp/a/eff14afa5cc4431298d457ec68b20977 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/a/eff14afa5cc4431298d457ec68b20977 2024-12-01T23:28:24,099 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/a/eff14afa5cc4431298d457ec68b20977, entries=2, sequenceid=2, filesize=5.0 K 2024-12-01T23:28:24,104 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d169d286ae05bbc58fe1bae2c16c5b1f in 519ms, sequenceid=2, compaction requested=false; wal=null 2024-12-01T23:28:24,106 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/0000000000000000002 2024-12-01T23:28:24,107 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:24,107 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:24,110 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d169d286ae05bbc58fe1bae2c16c5b1f 2024-12-01T23:28:24,114 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/d169d286ae05bbc58fe1bae2c16c5b1f/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-01T23:28:24,116 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d169d286ae05bbc58fe1bae2c16c5b1f; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62216960, jitterRate=-0.07289505004882812}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:28:24,117 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d169d286ae05bbc58fe1bae2c16c5b1f: Writing region info on filesystem at 1733095703528Initializing all the Stores at 1733095703534 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095703534Obtaining lock to block concurrent updates at 1733095703584 (+50 ms)Preparing flush snapshotting stores in d169d286ae05bbc58fe1bae2c16c5b1f at 1733095703584Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733095703587 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733095702274.d169d286ae05bbc58fe1bae2c16c5b1f. at 1733095703587Flushing d169d286ae05bbc58fe1bae2c16c5b1f/a: creating writer at 1733095703588 (+1 ms)Flushing d169d286ae05bbc58fe1bae2c16c5b1f/a: appending metadata at 1733095703621 (+33 ms)Flushing d169d286ae05bbc58fe1bae2c16c5b1f/a: closing flushed file at 1733095703623 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8c797c1: reopening flushed file at 1733095704088 (+465 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d169d286ae05bbc58fe1bae2c16c5b1f in 519ms, sequenceid=2, compaction requested=false; wal=null at 1733095704105 (+17 ms)Cleaning up temporary data from old regions at 1733095704107 (+2 ms)Region opened successfully at 1733095704116 (+9 ms) 2024-12-01T23:28:24,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=374 (was 360) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:47074 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@23d24ce[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@769bc067[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:60798 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:53222 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:60416 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:35981/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:60392 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=699 (was 605) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1179 (was 1239), ProcessCount=11 (was 11), AvailableMemoryMB=3852 (was 3650) - AvailableMemoryMB LEAK? - 2024-12-01T23:28:24,152 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=699, MaxFileDescriptor=1048576, SystemLoadAverage=1179, ProcessCount=11, AvailableMemoryMB=3852 2024-12-01T23:28:24,165 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:24,167 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:24,168 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:24,172 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-35836000, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/hregion-35836000, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:24,183 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-35836000/hregion-35836000.1733095704172, exclude list is [], retry=0 2024-12-01T23:28:24,187 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:24,188 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:24,188 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:24,215 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-35836000/hregion-35836000.1733095704172 2024-12-01T23:28:24,216 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:28:24,216 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 337d406d4b7ba3b2d5e630bc409a2fb0, NAME => 'testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/hbase 2024-12-01T23:28:24,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741846_1022 (size=64) 2024-12-01T23:28:24,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741846_1022 (size=64) 2024-12-01T23:28:24,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741846_1022 (size=64) 2024-12-01T23:28:24,232 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:24,234 INFO [StoreOpener-337d406d4b7ba3b2d5e630bc409a2fb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,237 INFO [StoreOpener-337d406d4b7ba3b2d5e630bc409a2fb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 337d406d4b7ba3b2d5e630bc409a2fb0 columnFamilyName a 2024-12-01T23:28:24,237 DEBUG [StoreOpener-337d406d4b7ba3b2d5e630bc409a2fb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:24,238 INFO [StoreOpener-337d406d4b7ba3b2d5e630bc409a2fb0-1 {}] regionserver.HStore(327): Store=337d406d4b7ba3b2d5e630bc409a2fb0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:24,238 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,239 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,240 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,241 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,241 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,244 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,248 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:24,249 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 337d406d4b7ba3b2d5e630bc409a2fb0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72615453, jitterRate=0.08205457031726837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:28:24,250 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 337d406d4b7ba3b2d5e630bc409a2fb0: Writing region info on filesystem at 1733095704233Initializing all the Stores at 1733095704234 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095704234Cleaning up temporary data from old regions at 1733095704241 (+7 ms)Region opened successfully at 1733095704249 (+8 ms) 2024-12-01T23:28:24,250 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 337d406d4b7ba3b2d5e630bc409a2fb0, disabling compactions & flushes 2024-12-01T23:28:24,250 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0. 2024-12-01T23:28:24,250 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0. 2024-12-01T23:28:24,250 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0. after waiting 0 ms 2024-12-01T23:28:24,250 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0. 2024-12-01T23:28:24,251 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0. 2024-12-01T23:28:24,251 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 337d406d4b7ba3b2d5e630bc409a2fb0: Waiting for close lock at 1733095704250Disabling compacts and flushes for region at 1733095704250Disabling writes for close at 1733095704250Writing region close event to WAL at 1733095704251 (+1 ms)Closed at 1733095704251 2024-12-01T23:28:24,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741845_1021 (size=175) 2024-12-01T23:28:24,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741845_1021 (size=175) 2024-12-01T23:28:24,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741845_1021 (size=175) 2024-12-01T23:28:24,259 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:24,259 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-35836000:(num 1733095704172) 2024-12-01T23:28:24,260 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T23:28:24,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741847_1023 (size=469) 2024-12-01T23:28:24,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741847_1023 (size=469) 2024-12-01T23:28:24,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741847_1023 (size=469) 2024-12-01T23:28:24,308 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T23:28:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741848_1024 (size=322) 2024-12-01T23:28:24,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741848_1024 (size=322) 2024-12-01T23:28:24,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741848_1024 (size=322) 2024-12-01T23:28:24,373 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-2, size=322 (322bytes) 2024-12-01T23:28:24,374 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-2 2024-12-01T23:28:24,375 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-2 after 1ms 2024-12-01T23:28:24,414 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:24,415 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-2 took 41ms 2024-12-01T23:28:24,423 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-2 so closing down 2024-12-01T23:28:24,423 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:24,425 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-01T23:28:24,452 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002-wal-2.temp 2024-12-01T23:28:24,453 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:24,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741849_1025 (size=322) 2024-12-01T23:28:24,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741849_1025 (size=322) 2024-12-01T23:28:24,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741849_1025 (size=322) 2024-12-01T23:28:24,462 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-01T23:28:24,464 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002 2024-12-01T23:28:24,464 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-01T23:28:24,465 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-2, journal: Splitting hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-2, size=322 (322bytes) at 1733095704374Finishing writing output for hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-2 so closing down at 1733095704423 (+49 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002-wal-2.temp at 1733095704452 (+29 ms)3 split writer threads finished at 1733095704453 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733095704462 (+9 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002 at 1733095704464 (+2 ms)Processed 1 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733095704465 (+1 ms) 2024-12-01T23:28:24,478 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-1, size=469 (469bytes) 2024-12-01T23:28:24,478 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-1 2024-12-01T23:28:24,479 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-1 after 1ms 2024-12-01T23:28:24,505 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:24,506 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-1 took 28ms 2024-12-01T23:28:24,509 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-1 so closing down 2024-12-01T23:28:24,509 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:24,511 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-01T23:28:24,541 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000001-wal-1.temp 2024-12-01T23:28:24,541 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:24,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741850_1026 (size=469) 2024-12-01T23:28:24,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741850_1026 (size=469) 2024-12-01T23:28:24,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741850_1026 (size=469) 2024-12-01T23:28:24,555 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-01T23:28:24,583 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:24,586 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002, length=322 2024-12-01T23:28:24,588 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002 2024-12-01T23:28:24,589 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 82 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-01T23:28:24,589 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-1, journal: Splitting hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-1, size=469 (469bytes) at 1733095704478Finishing writing output for hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-1 so closing down at 1733095704509 (+31 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000001-wal-1.temp at 1733095704541 (+32 ms)3 split writer threads finished at 1733095704541Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733095704555 (+14 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002 at 1733095704588 (+33 ms)Processed 2 edits across 1 Regions in 82 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733095704589 (+1 ms) 2024-12-01T23:28:24,589 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:24,591 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:24,604 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal.1733095704592, exclude list is [], retry=0 2024-12-01T23:28:24,607 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:24,608 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:24,608 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:24,641 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal.1733095704592 2024-12-01T23:28:24,642 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:24,642 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 337d406d4b7ba3b2d5e630bc409a2fb0, NAME => 'testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:24,642 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:24,642 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,642 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,644 INFO [StoreOpener-337d406d4b7ba3b2d5e630bc409a2fb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,647 INFO [StoreOpener-337d406d4b7ba3b2d5e630bc409a2fb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 337d406d4b7ba3b2d5e630bc409a2fb0 columnFamilyName a 2024-12-01T23:28:24,647 DEBUG [StoreOpener-337d406d4b7ba3b2d5e630bc409a2fb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:24,649 INFO [StoreOpener-337d406d4b7ba3b2d5e630bc409a2fb0-1 {}] regionserver.HStore(327): Store=337d406d4b7ba3b2d5e630bc409a2fb0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:24,649 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,650 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,652 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,654 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002 2024-12-01T23:28:24,685 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:24,687 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002 2024-12-01T23:28:24,687 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 337d406d4b7ba3b2d5e630bc409a2fb0 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-01T23:28:24,716 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/.tmp/a/55fb457c7bca41f9a06ef32f65cb5c31 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733095704259/Put/seqid=0 2024-12-01T23:28:24,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741852_1028 (size=5170) 2024-12-01T23:28:24,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741852_1028 (size=5170) 2024-12-01T23:28:24,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741852_1028 (size=5170) 2024-12-01T23:28:24,736 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/.tmp/a/55fb457c7bca41f9a06ef32f65cb5c31 2024-12-01T23:28:24,749 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/.tmp/a/55fb457c7bca41f9a06ef32f65cb5c31 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/a/55fb457c7bca41f9a06ef32f65cb5c31 2024-12-01T23:28:24,760 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/a/55fb457c7bca41f9a06ef32f65cb5c31, entries=2, sequenceid=2, filesize=5.0 K 2024-12-01T23:28:24,761 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 337d406d4b7ba3b2d5e630bc409a2fb0 in 73ms, sequenceid=2, compaction requested=false; wal=null 2024-12-01T23:28:24,762 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/0000000000000000002 2024-12-01T23:28:24,764 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,764 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,770 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 337d406d4b7ba3b2d5e630bc409a2fb0 2024-12-01T23:28:24,773 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/337d406d4b7ba3b2d5e630bc409a2fb0/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-01T23:28:24,775 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 337d406d4b7ba3b2d5e630bc409a2fb0; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66680914, jitterRate=-0.0063769519329071045}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:28:24,775 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 337d406d4b7ba3b2d5e630bc409a2fb0: Writing region info on filesystem at 1733095704642Initializing all the Stores at 1733095704644 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095704644Obtaining lock to block concurrent updates at 1733095704687 (+43 ms)Preparing flush snapshotting stores in 337d406d4b7ba3b2d5e630bc409a2fb0 at 1733095704687Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733095704688 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733095704166.337d406d4b7ba3b2d5e630bc409a2fb0. at 1733095704688Flushing 337d406d4b7ba3b2d5e630bc409a2fb0/a: creating writer at 1733095704688Flushing 337d406d4b7ba3b2d5e630bc409a2fb0/a: appending metadata at 1733095704714 (+26 ms)Flushing 337d406d4b7ba3b2d5e630bc409a2fb0/a: closing flushed file at 1733095704714Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49e65a51: reopening flushed file at 1733095704747 (+33 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 337d406d4b7ba3b2d5e630bc409a2fb0 in 73ms, sequenceid=2, compaction requested=false; wal=null at 1733095704761 (+14 ms)Cleaning up temporary data from old regions at 1733095704764 (+3 ms)Region opened successfully at 1733095704775 (+11 ms) 2024-12-01T23:28:24,795 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=384 (was 374) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:60488 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:60798 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:53270 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:60392 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:47146 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=781 (was 699) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1179 (was 1179), ProcessCount=11 (was 11), AvailableMemoryMB=3818 (was 3852) 2024-12-01T23:28:24,809 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=781, MaxFileDescriptor=1048576, SystemLoadAverage=1179, ProcessCount=11, AvailableMemoryMB=3818 2024-12-01T23:28:24,827 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:24,830 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:24,831 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:24,835 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45195543, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/hregion-45195543, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:24,849 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45195543/hregion-45195543.1733095704836, exclude list is [], retry=0 2024-12-01T23:28:24,853 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:24,853 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:24,854 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:24,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-01T23:28:24,894 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-01T23:28:24,897 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45195543/hregion-45195543.1733095704836 2024-12-01T23:28:24,900 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:28:24,901 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a4f2f139f21064cc26e36eeb75cc13eb, NAME => 'testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/hbase 2024-12-01T23:28:24,903 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T23:28:24,903 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-01T23:28:24,904 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-01T23:28:24,904 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-01T23:28:24,906 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-01T23:28:24,907 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-01T23:28:24,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741854_1030 (size=64) 2024-12-01T23:28:24,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741854_1030 (size=64) 2024-12-01T23:28:24,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741854_1030 (size=64) 2024-12-01T23:28:24,929 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:24,936 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:24,939 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4f2f139f21064cc26e36eeb75cc13eb columnFamilyName a 2024-12-01T23:28:24,939 DEBUG [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:24,940 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(327): Store=a4f2f139f21064cc26e36eeb75cc13eb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:24,940 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:24,943 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4f2f139f21064cc26e36eeb75cc13eb columnFamilyName b 2024-12-01T23:28:24,944 DEBUG [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:24,945 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(327): Store=a4f2f139f21064cc26e36eeb75cc13eb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:24,945 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:24,947 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4f2f139f21064cc26e36eeb75cc13eb columnFamilyName c 2024-12-01T23:28:24,947 DEBUG [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:24,948 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(327): Store=a4f2f139f21064cc26e36eeb75cc13eb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:24,948 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:24,949 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:24,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:24,953 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:24,953 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:24,954 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:24,956 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:24,959 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:24,960 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4f2f139f21064cc26e36eeb75cc13eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68303124, jitterRate=0.017795860767364502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:24,960 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4f2f139f21064cc26e36eeb75cc13eb: Writing region info on filesystem at 1733095704929Initializing all the Stores at 1733095704930 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095704930Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095704932 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095704932Cleaning up temporary data from old regions at 1733095704953 (+21 ms)Region opened successfully at 1733095704960 (+7 ms) 2024-12-01T23:28:24,960 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4f2f139f21064cc26e36eeb75cc13eb, disabling compactions & flushes 2024-12-01T23:28:24,961 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. 2024-12-01T23:28:24,961 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. 2024-12-01T23:28:24,961 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. after waiting 0 ms 2024-12-01T23:28:24,961 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. 2024-12-01T23:28:24,961 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. 2024-12-01T23:28:24,961 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4f2f139f21064cc26e36eeb75cc13eb: Waiting for close lock at 1733095704960Disabling compacts and flushes for region at 1733095704960Disabling writes for close at 1733095704961 (+1 ms)Writing region close event to WAL at 1733095704961Closed at 1733095704961 2024-12-01T23:28:24,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741853_1029 (size=175) 2024-12-01T23:28:24,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741853_1029 (size=175) 2024-12-01T23:28:24,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741853_1029 (size=175) 2024-12-01T23:28:24,983 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:24,983 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45195543:(num 1733095704836) 2024-12-01T23:28:24,983 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:24,987 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:25,001 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987, exclude list is [], retry=0 2024-12-01T23:28:25,004 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:25,005 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:25,005 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:25,038 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 2024-12-01T23:28:25,039 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:28:25,301 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987, size=0 (0bytes) 2024-12-01T23:28:25,301 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 might be still open, length is 0 2024-12-01T23:28:25,301 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 2024-12-01T23:28:25,302 WARN [IPC Server handler 1 on default port 35981 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-01T23:28:25,303 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 after 1ms 2024-12-01T23:28:26,127 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-01T23:28:27,050 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T23:28:28,276 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:47174 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41911:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47174 dst: /127.0.0.1:41911 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41911 remote=/127.0.0.1:47174]. Total timeout mills is 60000, 56986 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:28,278 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:60522 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44369:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60522 dst: /127.0.0.1:44369 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:28,284 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:53322 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53322 dst: /127.0.0.1:39219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:28,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741855_1032 (size=497775) 2024-12-01T23:28:28,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741855_1032 (size=497775) 2024-12-01T23:28:29,304 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 after 4002ms 2024-12-01T23:28:29,359 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:29,367 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 took 4066ms 2024-12-01T23:28:29,403 INFO [Time-limited test-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733095704987.temp 2024-12-01T23:28:29,459 INFO [Time-limited test-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000000001-wal.1733095704987.temp 2024-12-01T23:28:29,633 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987; continuing. 2024-12-01T23:28:29,633 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 so closing down 2024-12-01T23:28:29,633 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:29,635 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:29,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741856_1033 (size=497783) 2024-12-01T23:28:29,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741856_1033 (size=497783) 2024-12-01T23:28:29,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741856_1033 (size=497783) 2024-12-01T23:28:30,064 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000000001-wal.1733095704987.temp (wrote 3002 edits, skipped 0 edits in 47 ms) 2024-12-01T23:28:30,067 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000000001-wal.1733095704987.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000003002 2024-12-01T23:28:30,067 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 689 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T23:28:30,067 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987, journal: Splitting hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987, size=0 (0bytes) at 1733095705301Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000000001-wal.1733095704987.temp at 1733095709459 (+4158 ms)Split 1024 edits, skipped 0 edits. at 1733095709507 (+48 ms)Split 2048 edits, skipped 0 edits. at 1733095709591 (+84 ms)Finishing writing output for hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 so closing down at 1733095709633 (+42 ms)3 split writer threads finished at 1733095709635 (+2 ms)Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000000001-wal.1733095704987.temp (wrote 3002 edits, skipped 0 edits in 47 ms) at 1733095710064 (+429 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000000001-wal.1733095704987.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000003002 at 1733095710067 (+3 ms)Processed 3002 edits across 1 Regions in 689 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987, size=0, length=0, corrupted=false, cancelled=false at 1733095710067 2024-12-01T23:28:30,071 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 to hdfs://localhost:35981/hbase/oldWALs/wal.1733095704987 2024-12-01T23:28:30,073 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000003002 2024-12-01T23:28:30,073 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:30,077 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:30,097 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095710078, exclude list is [], retry=0 2024-12-01T23:28:30,105 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:30,105 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:30,106 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:30,173 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095710078 2024-12-01T23:28:30,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:28:30,177 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:30,187 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:30,188 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4f2f139f21064cc26e36eeb75cc13eb columnFamilyName a 2024-12-01T23:28:30,188 DEBUG [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:30,189 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(327): Store=a4f2f139f21064cc26e36eeb75cc13eb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:30,190 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:30,191 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4f2f139f21064cc26e36eeb75cc13eb columnFamilyName b 2024-12-01T23:28:30,192 DEBUG [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:30,193 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(327): Store=a4f2f139f21064cc26e36eeb75cc13eb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:30,193 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:30,195 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4f2f139f21064cc26e36eeb75cc13eb columnFamilyName c 2024-12-01T23:28:30,195 DEBUG [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:30,198 INFO [StoreOpener-a4f2f139f21064cc26e36eeb75cc13eb-1 {}] regionserver.HStore(327): Store=a4f2f139f21064cc26e36eeb75cc13eb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:30,199 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:30,201 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:30,225 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:30,226 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000003002 2024-12-01T23:28:30,284 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:30,360 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T23:28:30,994 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4f2f139f21064cc26e36eeb75cc13eb 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-01T23:28:31,108 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/a/454ed42356014fc39b37bd44f8637552 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733095705050/Put/seqid=0 2024-12-01T23:28:31,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741858_1035 (size=50463) 2024-12-01T23:28:31,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741858_1035 (size=50463) 2024-12-01T23:28:31,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741858_1035 (size=50463) 2024-12-01T23:28:31,189 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/a/454ed42356014fc39b37bd44f8637552 2024-12-01T23:28:31,213 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/a/454ed42356014fc39b37bd44f8637552 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/a/454ed42356014fc39b37bd44f8637552 2024-12-01T23:28:31,252 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/a/454ed42356014fc39b37bd44f8637552, entries=754, sequenceid=754, filesize=49.3 K 2024-12-01T23:28:31,252 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for a4f2f139f21064cc26e36eeb75cc13eb in 259ms, sequenceid=754, compaction requested=false; wal=null 2024-12-01T23:28:31,315 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T23:28:31,316 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4f2f139f21064cc26e36eeb75cc13eb 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-01T23:28:31,353 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/a/a0b8756095d94d1e9fc5e99ecbb78d7e is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733095705112/Put/seqid=0 2024-12-01T23:28:31,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741859_1036 (size=20072) 2024-12-01T23:28:31,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741859_1036 (size=20072) 2024-12-01T23:28:31,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741859_1036 (size=20072) 2024-12-01T23:28:31,445 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/a/a0b8756095d94d1e9fc5e99ecbb78d7e 2024-12-01T23:28:31,539 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/b/cc38f43ebfe147a88f40ca4265aa52c7 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733095705144/Put/seqid=0 2024-12-01T23:28:31,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741860_1037 (size=35835) 2024-12-01T23:28:31,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741860_1037 (size=35835) 2024-12-01T23:28:31,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741860_1037 (size=35835) 2024-12-01T23:28:31,631 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/b/cc38f43ebfe147a88f40ca4265aa52c7 2024-12-01T23:28:31,663 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/a/a0b8756095d94d1e9fc5e99ecbb78d7e as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/a/a0b8756095d94d1e9fc5e99ecbb78d7e 2024-12-01T23:28:31,695 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/a/a0b8756095d94d1e9fc5e99ecbb78d7e, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-01T23:28:31,700 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/b/cc38f43ebfe147a88f40ca4265aa52c7 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/b/cc38f43ebfe147a88f40ca4265aa52c7 2024-12-01T23:28:31,723 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/b/cc38f43ebfe147a88f40ca4265aa52c7, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-01T23:28:31,724 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for a4f2f139f21064cc26e36eeb75cc13eb in 408ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-01T23:28:31,742 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T23:28:31,743 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4f2f139f21064cc26e36eeb75cc13eb 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-01T23:28:31,769 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/b/e2626e99b4334bc6872dbbbeb60c26cb is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733095705169/Put/seqid=0 2024-12-01T23:28:31,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741861_1038 (size=35082) 2024-12-01T23:28:31,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741861_1038 (size=35082) 2024-12-01T23:28:31,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741861_1038 (size=35082) 2024-12-01T23:28:31,873 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/b/e2626e99b4334bc6872dbbbeb60c26cb 2024-12-01T23:28:31,921 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/c/a47b153895bf46898ed2cd849316adb7 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733095705209/Put/seqid=0 2024-12-01T23:28:31,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741862_1039 (size=20825) 2024-12-01T23:28:31,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741862_1039 (size=20825) 2024-12-01T23:28:31,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741862_1039 (size=20825) 2024-12-01T23:28:31,978 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/c/a47b153895bf46898ed2cd849316adb7 2024-12-01T23:28:32,014 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/b/e2626e99b4334bc6872dbbbeb60c26cb as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/b/e2626e99b4334bc6872dbbbeb60c26cb 2024-12-01T23:28:32,043 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/b/e2626e99b4334bc6872dbbbeb60c26cb, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-01T23:28:32,048 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/c/a47b153895bf46898ed2cd849316adb7 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/c/a47b153895bf46898ed2cd849316adb7 2024-12-01T23:28:32,092 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/c/a47b153895bf46898ed2cd849316adb7, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-01T23:28:32,093 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for a4f2f139f21064cc26e36eeb75cc13eb in 350ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-01T23:28:32,114 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733095705260/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. 2024-12-01T23:28:32,125 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000003002 2024-12-01T23:28:32,125 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T23:28:32,127 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4f2f139f21064cc26e36eeb75cc13eb 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-01T23:28:32,182 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/c/98b2cd5b80d84192bba457ead859b1c2 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733095705216/Put/seqid=0 2024-12-01T23:28:32,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741863_1040 (size=50301) 2024-12-01T23:28:32,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741863_1040 (size=50301) 2024-12-01T23:28:32,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741863_1040 (size=50301) 2024-12-01T23:28:32,285 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/c/98b2cd5b80d84192bba457ead859b1c2 2024-12-01T23:28:32,318 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 98b2cd5b80d84192bba457ead859b1c2 2024-12-01T23:28:32,328 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/.tmp/c/98b2cd5b80d84192bba457ead859b1c2 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/c/98b2cd5b80d84192bba457ead859b1c2 2024-12-01T23:28:32,352 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 98b2cd5b80d84192bba457ead859b1c2 2024-12-01T23:28:32,352 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/c/98b2cd5b80d84192bba457ead859b1c2, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-01T23:28:32,352 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for a4f2f139f21064cc26e36eeb75cc13eb in 225ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-01T23:28:32,353 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/0000000000000003002 2024-12-01T23:28:32,362 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:32,363 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:32,373 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-01T23:28:32,382 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4f2f139f21064cc26e36eeb75cc13eb 2024-12-01T23:28:32,389 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenIntoWAL/a4f2f139f21064cc26e36eeb75cc13eb/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-01T23:28:32,398 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4f2f139f21064cc26e36eeb75cc13eb; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75404824, jitterRate=0.1236194372177124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-01T23:28:32,399 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4f2f139f21064cc26e36eeb75cc13eb: Writing region info on filesystem at 1733095710178Initializing all the Stores at 1733095710186 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095710186Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095710186Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095710186Cleaning up temporary data from old regions at 1733095712363 (+2177 ms)Region opened successfully at 1733095712399 (+36 ms) 2024-12-01T23:28:32,522 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4f2f139f21064cc26e36eeb75cc13eb, disabling compactions & flushes 2024-12-01T23:28:32,523 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. 2024-12-01T23:28:32,523 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. 2024-12-01T23:28:32,523 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. after waiting 0 ms 2024-12-01T23:28:32,523 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. 2024-12-01T23:28:32,543 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733095704828.a4f2f139f21064cc26e36eeb75cc13eb. 2024-12-01T23:28:32,543 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4f2f139f21064cc26e36eeb75cc13eb: Waiting for close lock at 1733095712522Disabling compacts and flushes for region at 1733095712522Disabling writes for close at 1733095712523 (+1 ms)Writing region close event to WAL at 1733095712543 (+20 ms)Closed at 1733095712543 2024-12-01T23:28:32,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741857_1034 (size=175) 2024-12-01T23:28:32,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741857_1034 (size=175) 2024-12-01T23:28:32,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741857_1034 (size=175) 2024-12-01T23:28:32,568 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:32,568 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733095710078) 2024-12-01T23:28:32,599 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=397 (was 384) Potentially hanging thread: IPC Client (453912140) connection to localhost/127.0.0.1:42131 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1649104514_22 at /127.0.0.1:60532 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42391 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1649104514_22 at /127.0.0.1:47214 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1649104514_22 at /127.0.0.1:47232 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42131 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (453912140) connection to localhost/127.0.0.1:35981 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (453912140) connection to localhost/127.0.0.1:42391 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1649104514_22 at /127.0.0.1:53390 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:35981 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=859 (was 781) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1197 (was 1179) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3420 (was 3818) 2024-12-01T23:28:32,617 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=397, OpenFileDescriptor=859, MaxFileDescriptor=1048576, SystemLoadAverage=1197, ProcessCount=11, AvailableMemoryMB=3419 2024-12-01T23:28:32,640 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:32,644 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:32,645 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:32,657 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-06025196, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/hregion-06025196, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:32,674 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-06025196/hregion-06025196.1733095712657, exclude list is [], retry=0 2024-12-01T23:28:32,687 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:32,691 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:32,692 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:32,752 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-06025196/hregion-06025196.1733095712657 2024-12-01T23:28:32,753 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:28:32,753 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d64a1d5f79bc5debf2ac6f3012dfe954, NAME => 'test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/hbase 2024-12-01T23:28:32,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741865_1042 (size=43) 2024-12-01T23:28:32,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741865_1042 (size=43) 2024-12-01T23:28:32,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741865_1042 (size=43) 2024-12-01T23:28:32,788 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:32,790 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:32,792 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d64a1d5f79bc5debf2ac6f3012dfe954 columnFamilyName a 2024-12-01T23:28:32,792 DEBUG [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:32,796 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(327): Store=d64a1d5f79bc5debf2ac6f3012dfe954/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:32,796 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:32,799 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d64a1d5f79bc5debf2ac6f3012dfe954 columnFamilyName b 2024-12-01T23:28:32,799 DEBUG [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:32,800 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(327): Store=d64a1d5f79bc5debf2ac6f3012dfe954/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:32,800 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:32,802 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d64a1d5f79bc5debf2ac6f3012dfe954 columnFamilyName c 2024-12-01T23:28:32,802 DEBUG [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:32,802 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(327): Store=d64a1d5f79bc5debf2ac6f3012dfe954/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:32,803 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:32,803 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:32,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:32,806 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:32,806 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:32,806 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:32,808 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:32,813 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:32,813 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d64a1d5f79bc5debf2ac6f3012dfe954; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65775252, jitterRate=-0.01987236738204956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:32,814 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d64a1d5f79bc5debf2ac6f3012dfe954: Writing region info on filesystem at 1733095712788Initializing all the Stores at 1733095712790 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095712790Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095712790Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095712790Cleaning up temporary data from old regions at 1733095712806 (+16 ms)Region opened successfully at 1733095712814 (+8 ms) 2024-12-01T23:28:32,814 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d64a1d5f79bc5debf2ac6f3012dfe954, disabling compactions & flushes 2024-12-01T23:28:32,814 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. 2024-12-01T23:28:32,814 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. 2024-12-01T23:28:32,815 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. after waiting 0 ms 2024-12-01T23:28:32,815 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. 2024-12-01T23:28:32,815 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. 2024-12-01T23:28:32,815 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d64a1d5f79bc5debf2ac6f3012dfe954: Waiting for close lock at 1733095712814Disabling compacts and flushes for region at 1733095712814Disabling writes for close at 1733095712815 (+1 ms)Writing region close event to WAL at 1733095712815Closed at 1733095712815 2024-12-01T23:28:32,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741864_1041 (size=175) 2024-12-01T23:28:32,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741864_1041 (size=175) 2024-12-01T23:28:32,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741864_1041 (size=175) 2024-12-01T23:28:32,835 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:32,835 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-06025196:(num 1733095712657) 2024-12-01T23:28:32,836 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:32,838 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:32,856 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839, exclude list is [], retry=0 2024-12-01T23:28:32,860 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:32,861 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:32,861 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:32,904 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839 2024-12-01T23:28:32,905 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:28:33,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741866_1043 (size=371388) 2024-12-01T23:28:33,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741866_1043 (size=371388) 2024-12-01T23:28:33,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741866_1043 (size=371388) 2024-12-01T23:28:33,138 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839, size=362.7 K (371388bytes) 2024-12-01T23:28:33,138 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839 2024-12-01T23:28:33,139 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839 after 1ms 2024-12-01T23:28:33,181 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:33,183 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839 took 45ms 2024-12-01T23:28:33,187 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733095712839.temp 2024-12-01T23:28:33,225 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839 so closing down 2024-12-01T23:28:33,226 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:33,240 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000000001-wal.1733095712839.temp 2024-12-01T23:28:33,263 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:33,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741867_1044 (size=371388) 2024-12-01T23:28:33,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741867_1044 (size=371388) 2024-12-01T23:28:33,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741867_1044 (size=371388) 2024-12-01T23:28:33,679 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000000001-wal.1733095712839.temp (wrote 3000 edits, skipped 0 edits in 23 ms) 2024-12-01T23:28:33,681 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000000001-wal.1733095712839.temp to hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003000 2024-12-01T23:28:33,682 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 497 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-01T23:28:33,682 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839, journal: Splitting hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839, size=362.7 K (371388bytes) at 1733095713138Split 1024 edits, skipped 0 edits. at 1733095713197 (+59 ms)Split 2048 edits, skipped 0 edits. at 1733095713209 (+12 ms)Finishing writing output for hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839 so closing down at 1733095713226 (+17 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000000001-wal.1733095712839.temp at 1733095713240 (+14 ms)3 split writer threads finished at 1733095713264 (+24 ms)Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000000001-wal.1733095712839.temp (wrote 3000 edits, skipped 0 edits in 23 ms) at 1733095713679 (+415 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000000001-wal.1733095712839.temp to hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003000 at 1733095713681 (+2 ms)Processed 3000 edits across 1 Regions in 497 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733095713682 (+1 ms) 2024-12-01T23:28:33,684 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095712839 to hdfs://localhost:35981/hbase/oldWALs/wal.1733095712839 2024-12-01T23:28:33,685 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003000 2024-12-01T23:28:33,686 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:33,688 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:33,700 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688, exclude list is [], retry=0 2024-12-01T23:28:33,704 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:33,704 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:33,705 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:33,732 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688 2024-12-01T23:28:33,732 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:33,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741868_1045 (size=371515) 2024-12-01T23:28:33,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741868_1045 (size=371515) 2024-12-01T23:28:33,921 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688 not finished, retry = 0 2024-12-01T23:28:33,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741868_1045 (size=371515) 2024-12-01T23:28:34,062 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688, size=362.8 K (371515bytes) 2024-12-01T23:28:34,062 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688 2024-12-01T23:28:34,063 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688 after 1ms 2024-12-01T23:28:34,101 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:34,103 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688 took 41ms 2024-12-01T23:28:34,110 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733095713688.temp 2024-12-01T23:28:34,143 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688 so closing down 2024-12-01T23:28:34,143 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:34,149 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003001-wal.1733095713688.temp 2024-12-01T23:28:34,167 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:34,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741869_1046 (size=371515) 2024-12-01T23:28:34,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741869_1046 (size=371515) 2024-12-01T23:28:34,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741869_1046 (size=371515) 2024-12-01T23:28:34,236 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003001-wal.1733095713688.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-12-01T23:28:34,260 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003001-wal.1733095713688.temp to hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000006000 2024-12-01T23:28:34,264 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 160 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-01T23:28:34,264 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688, journal: Splitting hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688, size=362.8 K (371515bytes) at 1733095714062Split 1024 edits, skipped 0 edits. at 1733095714118 (+56 ms)Split 2048 edits, skipped 0 edits. at 1733095714133 (+15 ms)Finishing writing output for hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688 so closing down at 1733095714143 (+10 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003001-wal.1733095713688.temp at 1733095714149 (+6 ms)3 split writer threads finished at 1733095714167 (+18 ms)Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003001-wal.1733095713688.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1733095714236 (+69 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003001-wal.1733095713688.temp to hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000006000 at 1733095714260 (+24 ms)Processed 3000 edits across 1 Regions in 160 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733095714264 (+4 ms) 2024-12-01T23:28:34,293 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095713688 to hdfs://localhost:35981/hbase/oldWALs/wal.1733095713688 2024-12-01T23:28:34,299 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000006000 2024-12-01T23:28:34,299 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:34,307 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/test2727-manual,16010,1733095712638, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:34,328 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095714308, exclude list is [], retry=0 2024-12-01T23:28:34,339 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:34,339 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:34,340 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:34,378 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733095712638/wal.1733095714308 2024-12-01T23:28:34,378 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:28:34,379 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d64a1d5f79bc5debf2ac6f3012dfe954, NAME => 'test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:34,379 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:34,379 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:34,379 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:34,384 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:34,387 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d64a1d5f79bc5debf2ac6f3012dfe954 columnFamilyName a 2024-12-01T23:28:34,388 DEBUG [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:34,389 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(327): Store=d64a1d5f79bc5debf2ac6f3012dfe954/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:34,389 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:34,391 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d64a1d5f79bc5debf2ac6f3012dfe954 columnFamilyName b 2024-12-01T23:28:34,391 DEBUG [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:34,393 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(327): Store=d64a1d5f79bc5debf2ac6f3012dfe954/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:34,393 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:34,395 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d64a1d5f79bc5debf2ac6f3012dfe954 columnFamilyName c 2024-12-01T23:28:34,395 DEBUG [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:34,408 INFO [StoreOpener-d64a1d5f79bc5debf2ac6f3012dfe954-1 {}] regionserver.HStore(327): Store=d64a1d5f79bc5debf2ac6f3012dfe954/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:34,409 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:34,410 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:34,414 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:34,416 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003000 2024-12-01T23:28:34,466 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:34,569 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003000 2024-12-01T23:28:34,571 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000006000 2024-12-01T23:28:34,621 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:34,665 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000006000 2024-12-01T23:28:34,665 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d64a1d5f79bc5debf2ac6f3012dfe954 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-01T23:28:34,716 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/.tmp/a/f4b53ce4bc90416a87a2f4c5fd330a54 is 41, key is test2727/a:100/1733095713736/Put/seqid=0 2024-12-01T23:28:34,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741871_1048 (size=84227) 2024-12-01T23:28:34,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741871_1048 (size=84227) 2024-12-01T23:28:34,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741871_1048 (size=84227) 2024-12-01T23:28:34,808 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/.tmp/a/f4b53ce4bc90416a87a2f4c5fd330a54 2024-12-01T23:28:34,896 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/.tmp/b/f4efef848ec6403380791f0599928807 is 41, key is test2727/b:100/1733095713781/Put/seqid=0 2024-12-01T23:28:34,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741872_1049 (size=84609) 2024-12-01T23:28:34,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741872_1049 (size=84609) 2024-12-01T23:28:34,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741872_1049 (size=84609) 2024-12-01T23:28:35,355 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/.tmp/b/f4efef848ec6403380791f0599928807 2024-12-01T23:28:35,393 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/.tmp/c/c0adee175e3a4f5ebb7bc27518249d65 is 41, key is test2727/c:100/1733095713841/Put/seqid=0 2024-12-01T23:28:35,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741873_1050 (size=84609) 2024-12-01T23:28:35,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741873_1050 (size=84609) 2024-12-01T23:28:35,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741873_1050 (size=84609) 2024-12-01T23:28:35,448 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/.tmp/c/c0adee175e3a4f5ebb7bc27518249d65 2024-12-01T23:28:35,471 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/.tmp/a/f4b53ce4bc90416a87a2f4c5fd330a54 as hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/a/f4b53ce4bc90416a87a2f4c5fd330a54 2024-12-01T23:28:35,480 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/a/f4b53ce4bc90416a87a2f4c5fd330a54, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-01T23:28:35,481 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/.tmp/b/f4efef848ec6403380791f0599928807 as hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/b/f4efef848ec6403380791f0599928807 2024-12-01T23:28:35,492 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/b/f4efef848ec6403380791f0599928807, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-01T23:28:35,494 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/.tmp/c/c0adee175e3a4f5ebb7bc27518249d65 as hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/c/c0adee175e3a4f5ebb7bc27518249d65 2024-12-01T23:28:35,507 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/c/c0adee175e3a4f5ebb7bc27518249d65, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-01T23:28:35,508 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for d64a1d5f79bc5debf2ac6f3012dfe954 in 842ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-01T23:28:35,510 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000003000 2024-12-01T23:28:35,515 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/0000000000000006000 2024-12-01T23:28:35,524 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:35,524 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:35,526 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:35,529 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d64a1d5f79bc5debf2ac6f3012dfe954 2024-12-01T23:28:35,540 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/test2727/d64a1d5f79bc5debf2ac6f3012dfe954/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-01T23:28:35,542 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d64a1d5f79bc5debf2ac6f3012dfe954; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64955222, jitterRate=-0.032091766595840454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:35,543 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d64a1d5f79bc5debf2ac6f3012dfe954: Writing region info on filesystem at 1733095714379Initializing all the Stores at 1733095714384 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095714384Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095714384Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095714384Obtaining lock to block concurrent updates at 1733095714665 (+281 ms)Preparing flush snapshotting stores in d64a1d5f79bc5debf2ac6f3012dfe954 at 1733095714665Finished memstore snapshotting test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733095714666 (+1 ms)Flushing stores of test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. at 1733095714666Flushing d64a1d5f79bc5debf2ac6f3012dfe954/a: creating writer at 1733095714666Flushing d64a1d5f79bc5debf2ac6f3012dfe954/a: appending metadata at 1733095714715 (+49 ms)Flushing d64a1d5f79bc5debf2ac6f3012dfe954/a: closing flushed file at 1733095714715Flushing d64a1d5f79bc5debf2ac6f3012dfe954/b: creating writer at 1733095714844 (+129 ms)Flushing d64a1d5f79bc5debf2ac6f3012dfe954/b: appending metadata at 1733095714894 (+50 ms)Flushing d64a1d5f79bc5debf2ac6f3012dfe954/b: closing flushed file at 1733095714894Flushing d64a1d5f79bc5debf2ac6f3012dfe954/c: creating writer at 1733095715363 (+469 ms)Flushing d64a1d5f79bc5debf2ac6f3012dfe954/c: appending metadata at 1733095715392 (+29 ms)Flushing d64a1d5f79bc5debf2ac6f3012dfe954/c: closing flushed file at 1733095715392Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33f3fc4d: reopening flushed file at 1733095715469 (+77 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5533f66a: reopening flushed file at 1733095715480 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5df0a7d2: reopening flushed file at 1733095715492 (+12 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for d64a1d5f79bc5debf2ac6f3012dfe954 in 842ms, sequenceid=6000, compaction requested=false; wal=null at 1733095715508 (+16 ms)Cleaning up temporary data from old regions at 1733095715524 (+16 ms)Region opened successfully at 1733095715543 (+19 ms) 2024-12-01T23:28:35,545 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-01T23:28:35,545 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d64a1d5f79bc5debf2ac6f3012dfe954, disabling compactions & flushes 2024-12-01T23:28:35,545 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. 2024-12-01T23:28:35,545 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. 2024-12-01T23:28:35,545 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. after waiting 0 ms 2024-12-01T23:28:35,545 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. 2024-12-01T23:28:35,573 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733095712641.d64a1d5f79bc5debf2ac6f3012dfe954. 2024-12-01T23:28:35,573 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d64a1d5f79bc5debf2ac6f3012dfe954: Waiting for close lock at 1733095715545Disabling compacts and flushes for region at 1733095715545Disabling writes for close at 1733095715545Writing region close event to WAL at 1733095715573 (+28 ms)Closed at 1733095715573 2024-12-01T23:28:35,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741870_1047 (size=175) 2024-12-01T23:28:35,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741870_1047 (size=175) 2024-12-01T23:28:35,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741870_1047 (size=175) 2024-12-01T23:28:35,584 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:35,584 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733095714308) 2024-12-01T23:28:35,601 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=402 (was 397) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:35688 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:47948 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=923 (was 859) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1149 (was 1197), ProcessCount=11 (was 11), AvailableMemoryMB=3626 (was 3419) - AvailableMemoryMB LEAK? - 2024-12-01T23:28:35,616 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=402, OpenFileDescriptor=923, MaxFileDescriptor=1048576, SystemLoadAverage=1149, ProcessCount=11, AvailableMemoryMB=3641 2024-12-01T23:28:35,638 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:35,646 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:35,647 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733095715646 2024-12-01T23:28:35,682 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646 2024-12-01T23:28:35,688 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:28:35,690 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e78469d6fcc30e3e64ba0cd17af04af, NAME => 'testSequentialEditLogSeqNum,,1733095715639.8e78469d6fcc30e3e64ba0cd17af04af.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:35,691 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733095715639.8e78469d6fcc30e3e64ba0cd17af04af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:35,691 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8e78469d6fcc30e3e64ba0cd17af04af 2024-12-01T23:28:35,691 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8e78469d6fcc30e3e64ba0cd17af04af 2024-12-01T23:28:35,692 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af doesn't exist for region: 8e78469d6fcc30e3e64ba0cd17af04af on table testSequentialEditLogSeqNum 2024-12-01T23:28:35,693 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 8e78469d6fcc30e3e64ba0cd17af04af on table testSequentialEditLogSeqNum 2024-12-01T23:28:35,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741875_1052 (size=62) 2024-12-01T23:28:35,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741875_1052 (size=62) 2024-12-01T23:28:35,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741875_1052 (size=62) 2024-12-01T23:28:35,749 INFO [StoreOpener-8e78469d6fcc30e3e64ba0cd17af04af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8e78469d6fcc30e3e64ba0cd17af04af 2024-12-01T23:28:35,752 INFO [StoreOpener-8e78469d6fcc30e3e64ba0cd17af04af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e78469d6fcc30e3e64ba0cd17af04af columnFamilyName a 2024-12-01T23:28:35,752 DEBUG [StoreOpener-8e78469d6fcc30e3e64ba0cd17af04af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:35,753 INFO [StoreOpener-8e78469d6fcc30e3e64ba0cd17af04af-1 {}] regionserver.HStore(327): Store=8e78469d6fcc30e3e64ba0cd17af04af/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:35,753 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8e78469d6fcc30e3e64ba0cd17af04af 2024-12-01T23:28:35,754 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af 2024-12-01T23:28:35,755 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af 2024-12-01T23:28:35,755 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8e78469d6fcc30e3e64ba0cd17af04af 2024-12-01T23:28:35,756 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8e78469d6fcc30e3e64ba0cd17af04af 2024-12-01T23:28:35,761 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8e78469d6fcc30e3e64ba0cd17af04af 2024-12-01T23:28:35,764 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:35,765 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8e78469d6fcc30e3e64ba0cd17af04af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58879939, jitterRate=-0.12262053787708282}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:28:35,766 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8e78469d6fcc30e3e64ba0cd17af04af: Writing region info on filesystem at 1733095715691Initializing all the Stores at 1733095715748 (+57 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095715748Cleaning up temporary data from old regions at 1733095715756 (+8 ms)Region opened successfully at 1733095715766 (+10 ms) 2024-12-01T23:28:35,799 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8e78469d6fcc30e3e64ba0cd17af04af 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-01T23:28:35,835 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/.tmp/a/eabf0b57bf8147cbb877d900cc0080ce is 81, key is testSequentialEditLogSeqNum/a:x0/1733095715766/Put/seqid=0 2024-12-01T23:28:35,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741876_1053 (size=5833) 2024-12-01T23:28:35,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741876_1053 (size=5833) 2024-12-01T23:28:35,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741876_1053 (size=5833) 2024-12-01T23:28:35,864 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/.tmp/a/eabf0b57bf8147cbb877d900cc0080ce 2024-12-01T23:28:35,872 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/.tmp/a/eabf0b57bf8147cbb877d900cc0080ce as hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/a/eabf0b57bf8147cbb877d900cc0080ce 2024-12-01T23:28:35,881 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/a/eabf0b57bf8147cbb877d900cc0080ce, entries=10, sequenceid=13, filesize=5.7 K 2024-12-01T23:28:35,883 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 8e78469d6fcc30e3e64ba0cd17af04af in 83ms, sequenceid=13, compaction requested=false 2024-12-01T23:28:35,883 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8e78469d6fcc30e3e64ba0cd17af04af: 2024-12-01T23:28:35,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T23:28:35,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T23:28:35,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T23:28:35,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T23:28:35,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T23:28:35,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741874_1051 (size=3256) 2024-12-01T23:28:35,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741874_1051 (size=3256) 2024-12-01T23:28:35,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741874_1051 (size=3256) 2024-12-01T23:28:35,926 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646, size=3.2 K (3256bytes) 2024-12-01T23:28:35,926 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646 2024-12-01T23:28:35,927 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646 after 1ms 2024-12-01T23:28:35,983 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:35,983 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646 took 58ms 2024-12-01T23:28:36,001 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646 so closing down 2024-12-01T23:28:36,001 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:36,009 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733095715646.temp 2024-12-01T23:28:36,049 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/recovered.edits/0000000000000000003-wal.1733095715646.temp 2024-12-01T23:28:36,049 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:36,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741877_1054 (size=2650) 2024-12-01T23:28:36,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741877_1054 (size=2650) 2024-12-01T23:28:36,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741877_1054 (size=2650) 2024-12-01T23:28:36,068 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/recovered.edits/0000000000000000003-wal.1733095715646.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-01T23:28:36,077 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/recovered.edits/0000000000000000003-wal.1733095715646.temp to hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/recovered.edits/0000000000000000020 2024-12-01T23:28:36,077 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 85 ms; skipped=2; WAL=hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-01T23:28:36,077 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646, journal: Splitting hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646, size=3.2 K (3256bytes) at 1733095715926Finishing writing output for hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646 so closing down at 1733095716001 (+75 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/recovered.edits/0000000000000000003-wal.1733095715646.temp at 1733095716049 (+48 ms)3 split writer threads finished at 1733095716049Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/recovered.edits/0000000000000000003-wal.1733095715646.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733095716068 (+19 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/recovered.edits/0000000000000000003-wal.1733095715646.temp to hdfs://localhost:35981/hbase/data/default/testSequentialEditLogSeqNum/8e78469d6fcc30e3e64ba0cd17af04af/recovered.edits/0000000000000000020 at 1733095716077 (+9 ms)Processed 17 edits across 1 Regions in 85 ms; skipped=2; WAL=hdfs://localhost:35981/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733095715637/wal.1733095715646, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733095716077 2024-12-01T23:28:36,113 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=407 (was 402) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:47948 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=961 (was 923) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1149 (was 1149), ProcessCount=11 (was 11), AvailableMemoryMB=3577 (was 3641) 2024-12-01T23:28:36,138 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=407, OpenFileDescriptor=961, MaxFileDescriptor=1048576, SystemLoadAverage=1149, ProcessCount=11, AvailableMemoryMB=3571 2024-12-01T23:28:36,160 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:36,174 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:36,201 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:36,214 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-25814837, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/hregion-25814837, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:36,234 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-25814837/hregion-25814837.1733095716214, exclude list is [], retry=0 2024-12-01T23:28:36,239 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:36,239 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:36,241 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:36,300 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-25814837/hregion-25814837.1733095716214 2024-12-01T23:28:36,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:36,305 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => de217b5e7ba28d43e99ba804e7f7fd38, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/hbase 2024-12-01T23:28:36,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741879_1056 (size=70) 2024-12-01T23:28:36,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741879_1056 (size=70) 2024-12-01T23:28:36,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741879_1056 (size=70) 2024-12-01T23:28:36,328 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:36,332 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,341 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de217b5e7ba28d43e99ba804e7f7fd38 columnFamilyName a 2024-12-01T23:28:36,341 DEBUG [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:36,342 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(327): Store=de217b5e7ba28d43e99ba804e7f7fd38/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:36,342 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,345 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de217b5e7ba28d43e99ba804e7f7fd38 columnFamilyName b 2024-12-01T23:28:36,346 DEBUG [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:36,346 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(327): Store=de217b5e7ba28d43e99ba804e7f7fd38/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:36,347 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,349 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de217b5e7ba28d43e99ba804e7f7fd38 columnFamilyName c 2024-12-01T23:28:36,349 DEBUG [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:36,351 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(327): Store=de217b5e7ba28d43e99ba804e7f7fd38/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:36,351 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,353 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,353 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,355 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,355 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,356 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:36,357 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,365 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:36,366 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de217b5e7ba28d43e99ba804e7f7fd38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68059969, jitterRate=0.014172568917274475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:36,367 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de217b5e7ba28d43e99ba804e7f7fd38: Writing region info on filesystem at 1733095716328Initializing all the Stores at 1733095716331 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095716331Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095716332 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095716332Cleaning up temporary data from old regions at 1733095716355 (+23 ms)Region opened successfully at 1733095716367 (+12 ms) 2024-12-01T23:28:36,367 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing de217b5e7ba28d43e99ba804e7f7fd38, disabling compactions & flushes 2024-12-01T23:28:36,367 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. 2024-12-01T23:28:36,367 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. 2024-12-01T23:28:36,368 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. after waiting 0 ms 2024-12-01T23:28:36,368 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. 2024-12-01T23:28:36,369 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. 2024-12-01T23:28:36,369 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for de217b5e7ba28d43e99ba804e7f7fd38: Waiting for close lock at 1733095716367Disabling compacts and flushes for region at 1733095716367Disabling writes for close at 1733095716368 (+1 ms)Writing region close event to WAL at 1733095716369 (+1 ms)Closed at 1733095716369 2024-12-01T23:28:36,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741878_1055 (size=175) 2024-12-01T23:28:36,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741878_1055 (size=175) 2024-12-01T23:28:36,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741878_1055 (size=175) 2024-12-01T23:28:36,407 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:36,407 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-25814837:(num 1733095716214) 2024-12-01T23:28:36,407 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:36,412 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:36,431 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413, exclude list is [], retry=0 2024-12-01T23:28:36,438 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:36,438 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:36,439 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:36,482 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 2024-12-01T23:28:36,484 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:36,485 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => de217b5e7ba28d43e99ba804e7f7fd38, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:36,485 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:36,485 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,485 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,487 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,488 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de217b5e7ba28d43e99ba804e7f7fd38 columnFamilyName a 2024-12-01T23:28:36,488 DEBUG [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:36,489 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(327): Store=de217b5e7ba28d43e99ba804e7f7fd38/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:36,489 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,490 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de217b5e7ba28d43e99ba804e7f7fd38 columnFamilyName b 2024-12-01T23:28:36,490 DEBUG [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:36,491 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(327): Store=de217b5e7ba28d43e99ba804e7f7fd38/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:36,491 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,492 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de217b5e7ba28d43e99ba804e7f7fd38 columnFamilyName c 2024-12-01T23:28:36,492 DEBUG [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:36,493 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(327): Store=de217b5e7ba28d43e99ba804e7f7fd38/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:36,494 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,495 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,497 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,499 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,499 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,500 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:36,501 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:36,503 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de217b5e7ba28d43e99ba804e7f7fd38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71765737, jitterRate=0.06939281523227692}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:36,504 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de217b5e7ba28d43e99ba804e7f7fd38: Writing region info on filesystem at 1733095716485Initializing all the Stores at 1733095716486 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095716486Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095716487 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095716487Cleaning up temporary data from old regions at 1733095716499 (+12 ms)Region opened successfully at 1733095716504 (+5 ms) 2024-12-01T23:28:36,510 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733095716508/Put/seqid=0 2024-12-01T23:28:36,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741881_1058 (size=4826) 2024-12-01T23:28:36,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741881_1058 (size=4826) 2024-12-01T23:28:36,535 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35981/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in de217b5e7ba28d43e99ba804e7f7fd38/a 2024-12-01T23:28:36,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741881_1058 (size=4826) 2024-12-01T23:28:36,545 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-01T23:28:36,545 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T23:28:36,546 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for de217b5e7ba28d43e99ba804e7f7fd38: 2024-12-01T23:28:36,550 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/a/209fe793e2d04450921edd10e91138b6_SeqId_3_ 2024-12-01T23:28:36,551 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35981/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into de217b5e7ba28d43e99ba804e7f7fd38/a as hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/a/209fe793e2d04450921edd10e91138b6_SeqId_3_ - updating store file list. 2024-12-01T23:28:36,558 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 209fe793e2d04450921edd10e91138b6_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-01T23:28:36,559 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/a/209fe793e2d04450921edd10e91138b6_SeqId_3_ into de217b5e7ba28d43e99ba804e7f7fd38/a 2024-12-01T23:28:36,559 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35981/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into de217b5e7ba28d43e99ba804e7f7fd38/a (new location: hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/a/209fe793e2d04450921edd10e91138b6_SeqId_3_) 2024-12-01T23:28:36,629 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413, size=0 (0bytes) 2024-12-01T23:28:36,629 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 might be still open, length is 0 2024-12-01T23:28:36,629 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 2024-12-01T23:28:36,636 WARN [IPC Server handler 0 on default port 35981 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-01T23:28:36,637 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 after 8ms 2024-12-01T23:28:37,229 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:48026 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48026 dst: /127.0.0.1:39219 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39219 remote=/127.0.0.1:48026]. Total timeout mills is 60000, 59336 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:37,230 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:59192 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41911:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59192 dst: /127.0.0.1:41911 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:37,230 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:35782 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44369:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35782 dst: /127.0.0.1:44369 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:37,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741880_1059 (size=606) 2024-12-01T23:28:37,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741880_1059 (size=606) 2024-12-01T23:28:40,638 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 after 4009ms 2024-12-01T23:28:40,670 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:40,670 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 took 4042ms 2024-12-01T23:28:40,675 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413; continuing. 2024-12-01T23:28:40,675 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 so closing down 2024-12-01T23:28:40,675 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:40,677 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733095716413.temp 2024-12-01T23:28:40,717 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005-wal.1733095716413.temp 2024-12-01T23:28:40,718 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:40,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741882_1060 (size=329) 2024-12-01T23:28:40,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741882_1060 (size=329) 2024-12-01T23:28:40,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741882_1060 (size=329) 2024-12-01T23:28:40,752 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005-wal.1733095716413.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-01T23:28:40,756 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005-wal.1733095716413.temp to hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005 2024-12-01T23:28:40,756 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 85 ms; skipped=1; WAL=hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T23:28:40,756 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413, journal: Splitting hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413, size=0 (0bytes) at 1733095716629Finishing writing output for hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 so closing down at 1733095720675 (+4046 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005-wal.1733095716413.temp at 1733095720717 (+42 ms)3 split writer threads finished at 1733095720718 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005-wal.1733095716413.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733095720752 (+34 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005-wal.1733095716413.temp to hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005 at 1733095720756 (+4 ms)Processed 2 edits across 1 Regions in 85 ms; skipped=1; WAL=hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413, size=0, length=0, corrupted=false, cancelled=false at 1733095720756 2024-12-01T23:28:40,760 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 to hdfs://localhost:35981/hbase/oldWALs/wal.1733095716413 2024-12-01T23:28:40,762 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005 2024-12-01T23:28:40,762 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:40,765 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:40,784 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095720765, exclude list is [], retry=0 2024-12-01T23:28:40,789 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:40,789 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:40,790 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:40,852 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095720765 2024-12-01T23:28:40,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:28:40,857 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => de217b5e7ba28d43e99ba804e7f7fd38, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:40,857 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:40,857 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:40,857 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:40,866 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:40,869 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de217b5e7ba28d43e99ba804e7f7fd38 columnFamilyName a 2024-12-01T23:28:40,870 DEBUG [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:40,896 DEBUG [StoreFileOpener-de217b5e7ba28d43e99ba804e7f7fd38-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 209fe793e2d04450921edd10e91138b6_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-01T23:28:40,897 DEBUG [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/a/209fe793e2d04450921edd10e91138b6_SeqId_3_ 2024-12-01T23:28:40,897 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(327): Store=de217b5e7ba28d43e99ba804e7f7fd38/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:40,897 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:40,899 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de217b5e7ba28d43e99ba804e7f7fd38 columnFamilyName b 2024-12-01T23:28:40,899 DEBUG [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:40,902 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(327): Store=de217b5e7ba28d43e99ba804e7f7fd38/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:40,902 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:40,904 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de217b5e7ba28d43e99ba804e7f7fd38 columnFamilyName c 2024-12-01T23:28:40,904 DEBUG [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:40,905 INFO [StoreOpener-de217b5e7ba28d43e99ba804e7f7fd38-1 {}] regionserver.HStore(327): Store=de217b5e7ba28d43e99ba804e7f7fd38/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:40,905 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:40,906 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:40,912 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:40,913 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005 2024-12-01T23:28:40,961 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:40,963 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005 2024-12-01T23:28:40,963 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing de217b5e7ba28d43e99ba804e7f7fd38 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-01T23:28:40,990 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/.tmp/a/663374445d4f47d38343c4b959af4dde is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733095716565/Put/seqid=0 2024-12-01T23:28:41,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741884_1062 (size=5149) 2024-12-01T23:28:41,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741884_1062 (size=5149) 2024-12-01T23:28:41,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741884_1062 (size=5149) 2024-12-01T23:28:41,034 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/.tmp/a/663374445d4f47d38343c4b959af4dde 2024-12-01T23:28:41,071 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/.tmp/a/663374445d4f47d38343c4b959af4dde as hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/a/663374445d4f47d38343c4b959af4dde 2024-12-01T23:28:41,099 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/a/663374445d4f47d38343c4b959af4dde, entries=1, sequenceid=5, filesize=5.0 K 2024-12-01T23:28:41,099 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for de217b5e7ba28d43e99ba804e7f7fd38 in 136ms, sequenceid=5, compaction requested=false; wal=null 2024-12-01T23:28:41,100 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/0000000000000000005 2024-12-01T23:28:41,105 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:41,105 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:41,106 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:41,108 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de217b5e7ba28d43e99ba804e7f7fd38 2024-12-01T23:28:41,112 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/de217b5e7ba28d43e99ba804e7f7fd38/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-01T23:28:41,116 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de217b5e7ba28d43e99ba804e7f7fd38; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62187063, jitterRate=-0.07334055006504059}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:41,116 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de217b5e7ba28d43e99ba804e7f7fd38: Writing region info on filesystem at 1733095720857Initializing all the Stores at 1733095720864 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095720864Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095720865 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095720865Obtaining lock to block concurrent updates at 1733095720963 (+98 ms)Preparing flush snapshotting stores in de217b5e7ba28d43e99ba804e7f7fd38 at 1733095720963Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733095720963Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. at 1733095720963Flushing de217b5e7ba28d43e99ba804e7f7fd38/a: creating writer at 1733095720963Flushing de217b5e7ba28d43e99ba804e7f7fd38/a: appending metadata at 1733095720989 (+26 ms)Flushing de217b5e7ba28d43e99ba804e7f7fd38/a: closing flushed file at 1733095720989Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f8ef174: reopening flushed file at 1733095721067 (+78 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for de217b5e7ba28d43e99ba804e7f7fd38 in 136ms, sequenceid=5, compaction requested=false; wal=null at 1733095721099 (+32 ms)Cleaning up temporary data from old regions at 1733095721105 (+6 ms)Region opened successfully at 1733095721116 (+11 ms) 2024-12-01T23:28:41,124 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing de217b5e7ba28d43e99ba804e7f7fd38, disabling compactions & flushes 2024-12-01T23:28:41,124 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. 2024-12-01T23:28:41,124 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. 2024-12-01T23:28:41,125 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. after waiting 0 ms 2024-12-01T23:28:41,125 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. 2024-12-01T23:28:41,140 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733095716166.de217b5e7ba28d43e99ba804e7f7fd38. 2024-12-01T23:28:41,141 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for de217b5e7ba28d43e99ba804e7f7fd38: Waiting for close lock at 1733095721124Disabling compacts and flushes for region at 1733095721124Disabling writes for close at 1733095721125 (+1 ms)Writing region close event to WAL at 1733095721140 (+15 ms)Closed at 1733095721140 2024-12-01T23:28:41,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741883_1061 (size=175) 2024-12-01T23:28:41,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741883_1061 (size=175) 2024-12-01T23:28:41,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741883_1061 (size=175) 2024-12-01T23:28:41,163 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095720765 not finished, retry = 0 2024-12-01T23:28:41,266 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:41,266 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733095720765) 2024-12-01T23:28:41,279 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 407) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-90913630_22 at /127.0.0.1:48038 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-90913630_22 at /127.0.0.1:35812 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (453912140) connection to localhost/127.0.0.1:35981 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:35981 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-90913630_22 at /127.0.0.1:59234 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 961) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1081 (was 1149), ProcessCount=11 (was 11), AvailableMemoryMB=3524 (was 3571) 2024-12-01T23:28:41,281 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-01T23:28:41,295 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=1081, ProcessCount=11, AvailableMemoryMB=3524 2024-12-01T23:28:41,295 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-01T23:28:41,312 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:41,319 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T23:28:41,324 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f962de17fdc3,36351,1733095694462 2024-12-01T23:28:41,327 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5008967a 2024-12-01T23:28:41,329 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T23:28:41,331 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T23:28:41,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T23:28:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-01T23:28:41,358 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T23:28:41,361 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:41,367 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T23:28:41,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-01T23:28:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:28:41,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741885_1063 (size=694) 2024-12-01T23:28:41,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741885_1063 (size=694) 2024-12-01T23:28:41,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741885_1063 (size=694) 2024-12-01T23:28:41,470 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 153d48efbf4361bd73ec060241e37021, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0 2024-12-01T23:28:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:28:41,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741886_1064 (size=77) 2024-12-01T23:28:41,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741886_1064 (size=77) 2024-12-01T23:28:41,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741886_1064 (size=77) 2024-12-01T23:28:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:28:41,961 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:41,961 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 153d48efbf4361bd73ec060241e37021, disabling compactions & flushes 2024-12-01T23:28:41,961 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:41,961 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:41,961 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. after waiting 0 ms 2024-12-01T23:28:41,961 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:41,962 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:41,962 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 153d48efbf4361bd73ec060241e37021: Waiting for close lock at 1733095721961Disabling compacts and flushes for region at 1733095721961Disabling writes for close at 1733095721961Writing region close event to WAL at 1733095721961Closed at 1733095721961 2024-12-01T23:28:41,964 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T23:28:41,973 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733095721964"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733095721964"}]},"ts":"1733095721964"} 2024-12-01T23:28:41,978 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T23:28:41,980 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T23:28:41,986 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733095721980"}]},"ts":"1733095721980"} 2024-12-01T23:28:41,991 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-01T23:28:41,993 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f962de17fdc3=0} racks are {/default-rack=0} 2024-12-01T23:28:41,994 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T23:28:41,995 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T23:28:41,995 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-01T23:28:41,995 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T23:28:41,995 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T23:28:41,996 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-01T23:28:41,996 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T23:28:41,996 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T23:28:41,996 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-01T23:28:41,996 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T23:28:41,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, ASSIGN}] 2024-12-01T23:28:42,001 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, ASSIGN 2024-12-01T23:28:42,004 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, ASSIGN; state=OFFLINE, location=f962de17fdc3,39661,1733095695603; forceNewPlan=false, retain=false 2024-12-01T23:28:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:28:42,158 INFO [f962de17fdc3:36351 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T23:28:42,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=OPENING, regionLocation=f962de17fdc3,39661,1733095695603 2024-12-01T23:28:42,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, ASSIGN because future has completed 2024-12-01T23:28:42,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,39661,1733095695603}] 2024-12-01T23:28:42,335 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,336 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 153d48efbf4361bd73ec060241e37021, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:42,336 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,336 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:42,337 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,337 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,340 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,342 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf1 2024-12-01T23:28:42,342 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:42,348 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:42,349 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,351 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf2 2024-12-01T23:28:42,351 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:42,352 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:42,352 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,353 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,354 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,355 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,355 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,356 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T23:28:42,358 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,360 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:42,361 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 153d48efbf4361bd73ec060241e37021; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69059561, jitterRate=0.02906765043735504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T23:28:42,361 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,362 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 153d48efbf4361bd73ec060241e37021: Running coprocessor pre-open hook at 1733095722337Writing region info on filesystem at 1733095722337Initializing all the Stores at 1733095722338 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095722338Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095722339 (+1 ms)Cleaning up temporary data from old regions at 1733095722355 (+16 ms)Running coprocessor post-open hooks at 1733095722361 (+6 ms)Region opened successfully at 1733095722362 (+1 ms) 2024-12-01T23:28:42,369 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., pid=6, masterSystemTime=1733095722327 2024-12-01T23:28:42,372 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,373 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,374 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=OPEN, openSeqNum=2, regionLocation=f962de17fdc3,39661,1733095695603 2024-12-01T23:28:42,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,39661,1733095695603 because future has completed 2024-12-01T23:28:42,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-01T23:28:42,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,39661,1733095695603 in 220 msec 2024-12-01T23:28:42,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-01T23:28:42,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, ASSIGN in 399 msec 2024-12-01T23:28:42,402 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T23:28:42,402 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733095722402"}]},"ts":"1733095722402"} 2024-12-01T23:28:42,405 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-01T23:28:42,407 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T23:28:42,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 1.0680 sec 2024-12-01T23:28:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T23:28:42,522 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-01T23:28:42,522 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-01T23:28:42,524 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T23:28:42,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-01T23:28:42,536 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T23:28:42,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-01T23:28:42,554 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=2] 2024-12-01T23:28:42,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=153d48efbf4361bd73ec060241e37021, source=f962de17fdc3,39661,1733095695603, destination=f962de17fdc3,34145,1733095695383, warming up region on f962de17fdc3,34145,1733095695383 2024-12-01T23:28:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T23:28:42,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=153d48efbf4361bd73ec060241e37021, source=f962de17fdc3,39661,1733095695603, destination=f962de17fdc3,34145,1733095695383, running balancer 2024-12-01T23:28:42,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46563, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T23:28:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE 2024-12-01T23:28:42,607 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE 2024-12-01T23:28:42,615 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=CLOSING, regionLocation=f962de17fdc3,39661,1733095695603 2024-12-01T23:28:42,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] regionserver.HRegion(7855): Warmup {ENCODED => 153d48efbf4361bd73ec060241e37021, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:42,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE because future has completed 2024-12-01T23:28:42,632 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,640 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-01T23:28:42,640 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf1 2024-12-01T23:28:42,640 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:42,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,39661,1733095695603}] 2024-12-01T23:28:42,647 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:42,656 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-01T23:28:42,661 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf2 2024-12-01T23:28:42,661 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:42,665 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] regionserver.HRegion(1722): Closing 153d48efbf4361bd73ec060241e37021, disabling compactions & flushes 2024-12-01T23:28:42,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. after waiting 0 ms 2024-12-01T23:28:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] regionserver.HRegion(1676): Region close journal for 153d48efbf4361bd73ec060241e37021: Waiting for close lock at 1733095722665Disabling compacts and flushes for region at 1733095722665Disabling writes for close at 1733095722665Writing region close event to WAL at 1733095722666 (+1 ms)Closed at 1733095722666 2024-12-01T23:28:42,820 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:42,821 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-01T23:28:42,821 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 153d48efbf4361bd73ec060241e37021, disabling compactions & flushes 2024-12-01T23:28:42,822 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,822 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,822 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. after waiting 0 ms 2024-12-01T23:28:42,822 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:42,822 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 153d48efbf4361bd73ec060241e37021 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-01T23:28:42,897 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf1/660b43694e0449cf88d138b845261dd7 is 35, key is r1/cf1:q/1733095722558/Put/seqid=0 2024-12-01T23:28:43,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741887_1065 (size=4783) 2024-12-01T23:28:43,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741887_1065 (size=4783) 2024-12-01T23:28:43,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741887_1065 (size=4783) 2024-12-01T23:28:43,029 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf1/660b43694e0449cf88d138b845261dd7 2024-12-01T23:28:43,043 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf1/660b43694e0449cf88d138b845261dd7 as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/660b43694e0449cf88d138b845261dd7 2024-12-01T23:28:43,064 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/660b43694e0449cf88d138b845261dd7, entries=1, sequenceid=5, filesize=4.7 K 2024-12-01T23:28:43,073 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 153d48efbf4361bd73ec060241e37021 in 251ms, sequenceid=5, compaction requested=false 2024-12-01T23:28:43,073 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-01T23:28:43,128 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-01T23:28:43,139 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:43,140 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 153d48efbf4361bd73ec060241e37021: Waiting for close lock at 1733095722821Running coprocessor pre-close hooks at 1733095722821Disabling compacts and flushes for region at 1733095722821Disabling writes for close at 1733095722822 (+1 ms)Obtaining lock to block concurrent updates at 1733095722822Preparing flush snapshotting stores in 153d48efbf4361bd73ec060241e37021 at 1733095722822Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733095722822Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. at 1733095722832 (+10 ms)Flushing 153d48efbf4361bd73ec060241e37021/cf1: creating writer at 1733095722832Flushing 153d48efbf4361bd73ec060241e37021/cf1: appending metadata at 1733095722896 (+64 ms)Flushing 153d48efbf4361bd73ec060241e37021/cf1: closing flushed file at 1733095722897 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3229b51d: reopening flushed file at 1733095723042 (+145 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 153d48efbf4361bd73ec060241e37021 in 251ms, sequenceid=5, compaction requested=false at 1733095723073 (+31 ms)Writing region close event to WAL at 1733095723090 (+17 ms)Running coprocessor post-close hooks at 1733095723131 (+41 ms)Closed at 1733095723139 (+8 ms) 2024-12-01T23:28:43,141 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 153d48efbf4361bd73ec060241e37021 move to f962de17fdc3,34145,1733095695383 record at close sequenceid=5 2024-12-01T23:28:43,160 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,169 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=CLOSED 2024-12-01T23:28:43,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,39661,1733095695603 because future has completed 2024-12-01T23:28:43,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-01T23:28:43,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,39661,1733095695603 in 570 msec 2024-12-01T23:28:43,235 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE; state=CLOSED, location=f962de17fdc3,34145,1733095695383; forceNewPlan=false, retain=false 2024-12-01T23:28:43,309 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T23:28:43,387 INFO [f962de17fdc3:36351 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T23:28:43,387 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=OPENING, regionLocation=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:43,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE because future has completed 2024-12-01T23:28:43,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,34145,1733095695383}] 2024-12-01T23:28:43,576 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:43,577 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 153d48efbf4361bd73ec060241e37021, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:43,577 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,578 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:43,578 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,578 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,592 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,594 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf1 2024-12-01T23:28:43,594 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:43,607 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/660b43694e0449cf88d138b845261dd7 2024-12-01T23:28:43,608 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:43,608 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,612 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf2 2024-12-01T23:28:43,612 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:43,620 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:43,628 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,630 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,634 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,636 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,636 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,641 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T23:28:43,645 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,656 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 153d48efbf4361bd73ec060241e37021; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71998792, jitterRate=0.07286560535430908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T23:28:43,656 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:43,657 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 153d48efbf4361bd73ec060241e37021: Running coprocessor pre-open hook at 1733095723578Writing region info on filesystem at 1733095723578Initializing all the Stores at 1733095723588 (+10 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095723588Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095723592 (+4 ms)Cleaning up temporary data from old regions at 1733095723636 (+44 ms)Running coprocessor post-open hooks at 1733095723657 (+21 ms)Region opened successfully at 1733095723657 2024-12-01T23:28:43,668 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., pid=9, masterSystemTime=1733095723570 2024-12-01T23:28:43,682 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:43,682 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:43,689 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=OPEN, openSeqNum=9, regionLocation=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:43,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,34145,1733095695383 because future has completed 2024-12-01T23:28:43,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-01T23:28:43,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,34145,1733095695383 in 306 msec 2024-12-01T23:28:43,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE in 1.1330 sec 2024-12-01T23:28:43,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T23:28:43,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T23:28:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:44048 deadline: 1733095783803, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f962de17fdc3 port=34145 startCode=1733095695383. As of locationSeqNum=5. 2024-12-01T23:28:43,818 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f962de17fdc3 port=34145 startCode=1733095695383. As of locationSeqNum=5. 2024-12-01T23:28:43,819 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f962de17fdc3 port=34145 startCode=1733095695383. As of locationSeqNum=5. 2024-12-01T23:28:43,819 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,34145,1733095695383, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f962de17fdc3 port=34145 startCode=1733095695383. As of locationSeqNum=5. 2024-12-01T23:28:43,931 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T23:28:43,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T23:28:43,972 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 153d48efbf4361bd73ec060241e37021 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-01T23:28:44,035 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf1/2b488dd588bf47a89b9a2b78db4bbf78 is 29, key is r1/cf1:/1733095723937/DeleteFamily/seqid=0 2024-12-01T23:28:44,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741888_1066 (size=4906) 2024-12-01T23:28:44,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741888_1066 (size=4906) 2024-12-01T23:28:44,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741888_1066 (size=4906) 2024-12-01T23:28:44,097 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf1/2b488dd588bf47a89b9a2b78db4bbf78 2024-12-01T23:28:44,129 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b488dd588bf47a89b9a2b78db4bbf78 2024-12-01T23:28:44,179 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf2/eb4290bf804244d2bc9033aae7ead053 is 29, key is r1/cf2:/1733095723937/DeleteFamily/seqid=0 2024-12-01T23:28:44,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741889_1067 (size=4906) 2024-12-01T23:28:44,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741889_1067 (size=4906) 2024-12-01T23:28:44,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741889_1067 (size=4906) 2024-12-01T23:28:44,241 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf2/eb4290bf804244d2bc9033aae7ead053 2024-12-01T23:28:44,262 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eb4290bf804244d2bc9033aae7ead053 2024-12-01T23:28:44,268 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf1/2b488dd588bf47a89b9a2b78db4bbf78 as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/2b488dd588bf47a89b9a2b78db4bbf78 2024-12-01T23:28:44,287 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b488dd588bf47a89b9a2b78db4bbf78 2024-12-01T23:28:44,287 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/2b488dd588bf47a89b9a2b78db4bbf78, entries=1, sequenceid=12, filesize=4.8 K 2024-12-01T23:28:44,289 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf2/eb4290bf804244d2bc9033aae7ead053 as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/eb4290bf804244d2bc9033aae7ead053 2024-12-01T23:28:44,315 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eb4290bf804244d2bc9033aae7ead053 2024-12-01T23:28:44,315 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/eb4290bf804244d2bc9033aae7ead053, entries=1, sequenceid=12, filesize=4.8 K 2024-12-01T23:28:44,324 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 153d48efbf4361bd73ec060241e37021 in 352ms, sequenceid=12, compaction requested=false 2024-12-01T23:28:44,324 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 153d48efbf4361bd73ec060241e37021: 2024-12-01T23:28:44,328 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-01T23:28:44,330 DEBUG [Time-limited test {}] regionserver.HStore(1541): 153d48efbf4361bd73ec060241e37021/cf1 is initiating major compaction (all files) 2024-12-01T23:28:44,330 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T23:28:44,330 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:44,331 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 153d48efbf4361bd73ec060241e37021/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:44,331 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/660b43694e0449cf88d138b845261dd7, hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/2b488dd588bf47a89b9a2b78db4bbf78] into tmpdir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp, totalSize=9.5 K 2024-12-01T23:28:44,333 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 660b43694e0449cf88d138b845261dd7, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733095722558 2024-12-01T23:28:44,333 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2b488dd588bf47a89b9a2b78db4bbf78, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-01T23:28:44,357 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 153d48efbf4361bd73ec060241e37021#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-01T23:28:44,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741890_1068 (size=4626) 2024-12-01T23:28:44,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741890_1068 (size=4626) 2024-12-01T23:28:44,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741890_1068 (size=4626) 2024-12-01T23:28:44,468 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf1/51b849444c1c44f0b9c6c415a8b33faf as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/51b849444c1c44f0b9c6c415a8b33faf 2024-12-01T23:28:44,520 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 153d48efbf4361bd73ec060241e37021/cf1 of 153d48efbf4361bd73ec060241e37021 into 51b849444c1c44f0b9c6c415a8b33faf(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T23:28:44,520 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 153d48efbf4361bd73ec060241e37021: 2024-12-01T23:28:44,520 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-01T23:28:44,520 DEBUG [Time-limited test {}] regionserver.HStore(1541): 153d48efbf4361bd73ec060241e37021/cf2 is initiating major compaction (all files) 2024-12-01T23:28:44,521 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T23:28:44,521 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T23:28:44,521 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 153d48efbf4361bd73ec060241e37021/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:44,521 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/eb4290bf804244d2bc9033aae7ead053] into tmpdir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp, totalSize=4.8 K 2024-12-01T23:28:44,529 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting eb4290bf804244d2bc9033aae7ead053, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-01T23:28:44,559 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 153d48efbf4361bd73ec060241e37021#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-01T23:28:44,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741891_1069 (size=4592) 2024-12-01T23:28:44,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741891_1069 (size=4592) 2024-12-01T23:28:44,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741891_1069 (size=4592) 2024-12-01T23:28:44,696 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/.tmp/cf2/0569d9a3b0a8445ca2a76e0bf45e8f45 as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/0569d9a3b0a8445ca2a76e0bf45e8f45 2024-12-01T23:28:44,739 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 153d48efbf4361bd73ec060241e37021/cf2 of 153d48efbf4361bd73ec060241e37021 into 0569d9a3b0a8445ca2a76e0bf45e8f45(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T23:28:44,740 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 153d48efbf4361bd73ec060241e37021: 2024-12-01T23:28:44,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=153d48efbf4361bd73ec060241e37021, source=f962de17fdc3,34145,1733095695383, destination=f962de17fdc3,39661,1733095695603, warming up region on f962de17fdc3,39661,1733095695603 2024-12-01T23:28:44,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=153d48efbf4361bd73ec060241e37021, source=f962de17fdc3,34145,1733095695383, destination=f962de17fdc3,39661,1733095695603, running balancer 2024-12-01T23:28:44,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] regionserver.HRegion(7855): Warmup {ENCODED => 153d48efbf4361bd73ec060241e37021, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE 2024-12-01T23:28:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:44,768 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE 2024-12-01T23:28:44,770 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:44,781 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=CLOSING, regionLocation=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:44,781 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf1 2024-12-01T23:28:44,781 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:44,786 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=f962de17fdc3,34145,1733095695383, table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-01T23:28:44,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE because future has completed 2024-12-01T23:28:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-01T23:28:44,821 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-01T23:28:44,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,34145,1733095695383}] 2024-12-01T23:28:44,827 INFO [StoreFileOpener-153d48efbf4361bd73ec060241e37021-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b488dd588bf47a89b9a2b78db4bbf78 2024-12-01T23:28:44,828 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/2b488dd588bf47a89b9a2b78db4bbf78 2024-12-01T23:28:44,840 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/51b849444c1c44f0b9c6c415a8b33faf 2024-12-01T23:28:44,852 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/660b43694e0449cf88d138b845261dd7 2024-12-01T23:28:44,852 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:44,853 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:44,857 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf2 2024-12-01T23:28:44,857 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:44,881 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/0569d9a3b0a8445ca2a76e0bf45e8f45 2024-12-01T23:28:44,890 INFO [StoreFileOpener-153d48efbf4361bd73ec060241e37021-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eb4290bf804244d2bc9033aae7ead053 2024-12-01T23:28:44,890 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/eb4290bf804244d2bc9033aae7ead053 2024-12-01T23:28:44,891 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] regionserver.HRegion(1722): Closing 153d48efbf4361bd73ec060241e37021, disabling compactions & flushes 2024-12-01T23:28:44,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. after waiting 0 ms 2024-12-01T23:28:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:44,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39661 {}] regionserver.HRegion(1676): Region close journal for 153d48efbf4361bd73ec060241e37021: Waiting for close lock at 1733095724891Disabling compacts and flushes for region at 1733095724891Disabling writes for close at 1733095724891Writing region close event to WAL at 1733095724892 (+1 ms)Closed at 1733095724892 2024-12-01T23:28:44,896 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-01T23:28:44,896 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-01T23:28:44,898 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-01T23:28:44,898 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-01T23:28:44,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-01T23:28:44,900 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-01T23:28:44,986 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:44,986 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-01T23:28:44,986 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 153d48efbf4361bd73ec060241e37021, disabling compactions & flushes 2024-12-01T23:28:44,986 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:44,987 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:44,987 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. after waiting 0 ms 2024-12-01T23:28:44,987 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:44,996 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/660b43694e0449cf88d138b845261dd7, hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/2b488dd588bf47a89b9a2b78db4bbf78] to archive 2024-12-01T23:28:45,004 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T23:28:45,032 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/660b43694e0449cf88d138b845261dd7 to hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/660b43694e0449cf88d138b845261dd7 2024-12-01T23:28:45,064 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/2b488dd588bf47a89b9a2b78db4bbf78 to hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/2b488dd588bf47a89b9a2b78db4bbf78 2024-12-01T23:28:45,103 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/eb4290bf804244d2bc9033aae7ead053] to archive 2024-12-01T23:28:45,112 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T23:28:45,140 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/eb4290bf804244d2bc9033aae7ead053 to hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/eb4290bf804244d2bc9033aae7ead053 2024-12-01T23:28:45,196 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-01T23:28:45,198 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:45,198 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 153d48efbf4361bd73ec060241e37021: Waiting for close lock at 1733095724986Running coprocessor pre-close hooks at 1733095724986Disabling compacts and flushes for region at 1733095724986Disabling writes for close at 1733095724987 (+1 ms)Writing region close event to WAL at 1733095725164 (+177 ms)Running coprocessor post-close hooks at 1733095725198 (+34 ms)Closed at 1733095725198 2024-12-01T23:28:45,198 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 153d48efbf4361bd73ec060241e37021 move to f962de17fdc3,39661,1733095695603 record at close sequenceid=12 2024-12-01T23:28:45,202 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,208 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=CLOSED 2024-12-01T23:28:45,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,34145,1733095695383 because future has completed 2024-12-01T23:28:45,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-01T23:28:45,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,34145,1733095695383 in 419 msec 2024-12-01T23:28:45,276 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE; state=CLOSED, location=f962de17fdc3,39661,1733095695603; forceNewPlan=false, retain=false 2024-12-01T23:28:45,427 INFO [f962de17fdc3:36351 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T23:28:45,428 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=OPENING, regionLocation=f962de17fdc3,39661,1733095695603 2024-12-01T23:28:45,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE because future has completed 2024-12-01T23:28:45,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,39661,1733095695603}] 2024-12-01T23:28:45,590 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:45,590 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 153d48efbf4361bd73ec060241e37021, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:45,591 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,591 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:45,591 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,591 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,593 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,594 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf1 2024-12-01T23:28:45,595 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:45,603 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/51b849444c1c44f0b9c6c415a8b33faf 2024-12-01T23:28:45,603 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:45,603 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,604 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf2 2024-12-01T23:28:45,605 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:45,623 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/0569d9a3b0a8445ca2a76e0bf45e8f45 2024-12-01T23:28:45,624 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:45,624 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,626 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,629 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,630 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,630 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,632 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T23:28:45,634 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,636 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 153d48efbf4361bd73ec060241e37021; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65827616, jitterRate=-0.019092082977294922}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T23:28:45,636 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,637 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 153d48efbf4361bd73ec060241e37021: Running coprocessor pre-open hook at 1733095725591Writing region info on filesystem at 1733095725591Initializing all the Stores at 1733095725593 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095725593Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095725593Cleaning up temporary data from old regions at 1733095725630 (+37 ms)Running coprocessor post-open hooks at 1733095725636 (+6 ms)Region opened successfully at 1733095725637 (+1 ms) 2024-12-01T23:28:45,640 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., pid=12, masterSystemTime=1733095725585 2024-12-01T23:28:45,645 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:45,645 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:45,653 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=OPEN, openSeqNum=18, regionLocation=f962de17fdc3,39661,1733095695603 2024-12-01T23:28:45,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,39661,1733095695603 because future has completed 2024-12-01T23:28:45,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-01T23:28:45,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,39661,1733095695603 in 228 msec 2024-12-01T23:28:45,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, REOPEN/MOVE in 907 msec 2024-12-01T23:28:45,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T23:28:45,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T23:28:45,688 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server f962de17fdc3,39661,1733095695603: testing ***** 2024-12-01T23:28:45,688 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-01T23:28:45,691 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-01T23:28:45,693 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-01T23:28:45,699 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-01T23:28:45,709 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-01T23:28:45,733 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 443723776 }, "NonHeapMemoryUsage": { "committed": 172687360, "init": 7667712, "max": -1, "used": 170067880 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "f962de17fdc3", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 135, "ProcessCallTime_mean": 11, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 10, "ProcessCallTime_90th_percentile": 14, "ProcessCallTime_95th_percentile": 63, "ProcessCallTime_98th_percentile": 135, "ProcessCallTime_99th_percentile": 135, "ProcessCallTime_99.9th_percentile": 135, "ProcessCallTime_TimeRangeCount_0-1": 5, "ProcessCallTime_TimeRangeCount_1-3": 9, "ProcessCallTime_TimeRangeCount_3-10": 10, "ProcessCallTime_TimeRangeCount_10-30": 5, "ProcessCallTime_TimeRangeCount_30-100": 1, "ProcessCallTime_TimeRangeCount_100-300": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 3, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 1, "QueueCallTime_90th_percentile": 2, "QueueCallTime_95th_percentile": 3, "QueueCallTime_98th_percentile": 3, "QueueCallTime_99th_percentile": 3, "QueueCallTime_99.9th_percentile": 3, "QueueCallTime_TimeRangeCount_0-1": 25, "QueueCallTime_TimeRangeCount_1-3": 6, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 135, "TotalCallTime_mean": 12, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 4, "TotalCallTime_75th_percentile": 11, "TotalCallTime_90th_percentile": 15, "TotalCallTime_95th_percentile": 63, "TotalCallTime_98th_percentile": 135, "TotalCallTime_99th_percentile": 135, "TotalCallTime_99.9th_percentile": 135, "TotalCallTime_TimeRangeCount_0-1": 1, "TotalCallTime_TimeRangeCount_1-3": 11, "TotalCallTime_TimeRangeCount_3-10": 10, "TotalCallTime_TimeRangeCount_10-30": 7, "TotalCallTime_TimeRangeCount_30-100": 1, "TotalCallTime_TimeRangeCount_100-300": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 160, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 454, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1002, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 295, "RequestSize_90th_percentile": 352, "RequestSize_95th_percentile": 371, "RequestSize_98th_percentile": 382, "RequestSize_99th_percentile": 386, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_10-100": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5781 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "f962de17fdc3", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56655", "tag.serverName": "f962de17fdc3,39661,1733095695603", "tag.clusterId": "3b9d21d3-1a5d-4ed1-a1a3-dd57abfb96d0", "tag.Context": "regionserver", "tag.Hostname": "f962de17fdc3", "regionCount": 2, "storeCount": 4, "hlogFileCount": 10, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 4266, "memStoreHeapSize": 7584, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733095695603, "averageRegionSize": 4266, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.0, "writeRequestRatePerSecond": 2.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 599040, "ByteBuffAllocatorHeapAllocationRatio": 0.35775729671400613, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 29, "totalRowActionRequestCount": 24, "readRequestCount": 12, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 12, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 17, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "ScanTime_TimeRangeCount_1-3": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 16, "Put_min": 2, "Put_max": 14, "Put_mean": 6, "Put_25th_percentile": 4, "Put_median": 7, "Put_75th_percentile": 10, "Put_90th_percentile": 13, "Put_95th_percentile": 14, "Put_98th_percentile": 14, "Put_99th_percentile": 14, "Put_99.9th_percentile": 14, "Put_TimeRangeCount_0-1": 8, "Put_TimeRangeCount_3-10": 3, "Put_TimeRangeCount_10-30": 3, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 251, "FlushTime_max": 251, "FlushTime_mean": 251, "FlushTime_25th_percentile": 251, "FlushTime_median": 251, "FlushTime_75th_percentile": 251, "FlushTime_90th_percentile": 251, "FlushTime_95th_percentile": 251, "FlushTime_98th_percentile": 251, "FlushTime_99th_percentile": 251, "FlushTime_99.9th_percentile": 251, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 64, "ScanSize_max": 1112, "ScanSize_mean": 902, "ScanSize_25th_percentile": 308, "ScanSize_median": 576, "ScanSize_75th_percentile": 844, "ScanSize_90th_percentile": 1004, "ScanSize_95th_percentile": 1058, "ScanSize_98th_percentile": 1090, "ScanSize_99th_percentile": 1101, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_10-100": 1, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-01T23:28:45,745 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36351 {}] master.MasterRpcServices(700): f962de17fdc3,39661,1733095695603 reported a fatal error: ***** ABORTING region server f962de17fdc3,39661,1733095695603: testing ***** 2024-12-01T23:28:45,753 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f962de17fdc3,39661,1733095695603' ***** 2024-12-01T23:28:45,753 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-01T23:28:45,754 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T23:28:45,754 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T23:28:45,755 INFO [RS:2;f962de17fdc3:39661 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-01T23:28:45,755 INFO [RS:2;f962de17fdc3:39661 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-01T23:28:45,755 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(3091): Received CLOSE for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:45,755 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(956): aborting server f962de17fdc3,39661,1733095695603 2024-12-01T23:28:45,756 INFO [RS:2;f962de17fdc3:39661 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:28:45,756 INFO [RS:2;f962de17fdc3:39661 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f962de17fdc3:39661. 2024-12-01T23:28:45,756 DEBUG [RS:2;f962de17fdc3:39661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:28:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34145 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:48444 deadline: 1733095785756, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f962de17fdc3 port=39661 startCode=1733095695603. As of locationSeqNum=12. 2024-12-01T23:28:45,757 DEBUG [RS:2;f962de17fdc3:39661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:28:45,758 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,34145,1733095695383, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,34145,1733095695383, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f962de17fdc3 port=39661 startCode=1733095695603. As of locationSeqNum=12. 2024-12-01T23:28:45,758 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,34145,1733095695383, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f962de17fdc3 port=39661 startCode=1733095695603. As of locationSeqNum=12. 2024-12-01T23:28:45,758 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,34145,1733095695383, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=f962de17fdc3 port=39661 startCode=1733095695603. As of locationSeqNum=12. 2024-12-01T23:28:45,759 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T23:28:45,759 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T23:28:45,759 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T23:28:45,759 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T23:28:45,760 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-01T23:28:45,760 DEBUG [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 153d48efbf4361bd73ec060241e37021=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.} 2024-12-01T23:28:45,760 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 153d48efbf4361bd73ec060241e37021, disabling compactions & flushes 2024-12-01T23:28:45,761 DEBUG [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(1351): Waiting on 153d48efbf4361bd73ec060241e37021, 1588230740 2024-12-01T23:28:45,761 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:45,761 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:45,761 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. after waiting 0 ms 2024-12-01T23:28:45,761 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:45,761 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T23:28:45,761 INFO [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T23:28:45,761 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T23:28:45,761 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T23:28:45,761 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T23:28:45,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f962de17fdc3,39661,1733095695603 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:45,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f962de17fdc3,39661,1733095695603 aborting 2024-12-01T23:28:45,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f962de17fdc3,39661,1733095695603 aborting 2024-12-01T23:28:45,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 from cache 2024-12-01T23:28:45,769 ERROR [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-01T23:28:45,769 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T23:28:45,770 INFO [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T23:28:45,770 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733095725761Running coprocessor pre-close hooks at 1733095725761Disabling compacts and flushes for region at 1733095725761Disabling writes for close at 1733095725761Writing region close event to WAL at 1733095725769 (+8 ms)Running coprocessor post-close hooks at 1733095725769Closed at 1733095725770 (+1 ms) 2024-12-01T23:28:45,770 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T23:28:45,792 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:45,799 INFO [regionserver/f962de17fdc3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:28:45,804 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 153d48efbf4361bd73ec060241e37021: Waiting for close lock at 1733095725760Running coprocessor pre-close hooks at 1733095725760Disabling compacts and flushes for region at 1733095725760Disabling writes for close at 1733095725761 (+1 ms)Writing region close event to WAL at 1733095725792 (+31 ms)Running coprocessor post-close hooks at 1733095725792Closed at 1733095725792 2024-12-01T23:28:45,804 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:45,872 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:28:45,877 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1] 2024-12-01T23:28:45,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f962de17fdc3,39661,1733095695603 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:45,884 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f962de17fdc3,39661,1733095695603 aborting 2024-12-01T23:28:45,885 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f962de17fdc3,39661,1733095695603 aborting 2024-12-01T23:28:45,885 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 from cache 2024-12-01T23:28:45,961 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(976): stopping server f962de17fdc3,39661,1733095695603; all regions closed. 2024-12-01T23:28:45,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741836_1012 (size=7323) 2024-12-01T23:28:45,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741836_1012 (size=7323) 2024-12-01T23:28:45,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741836_1012 (size=7323) 2024-12-01T23:28:45,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741834_1010 (size=2288) 2024-12-01T23:28:45,997 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603/f962de17fdc3%2C39661%2C1733095695603.1733095700522 not finished, retry = 0 2024-12-01T23:28:45,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741834_1010 (size=2288) 2024-12-01T23:28:45,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741834_1010 (size=2288) 2024-12-01T23:28:46,091 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:28:46,093 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1] 2024-12-01T23:28:46,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f962de17fdc3,39661,1733095695603 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:46,095 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f962de17fdc3,39661,1733095695603 aborting 2024-12-01T23:28:46,095 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server f962de17fdc3,39661,1733095695603 aborting 2024-12-01T23:28:46,096 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 from cache 2024-12-01T23:28:46,098 DEBUG [RS:2;f962de17fdc3:39661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:28:46,098 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:28:46,098 INFO [RS:2;f962de17fdc3:39661 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:28:46,099 INFO [RS:2;f962de17fdc3:39661 {}] hbase.ChoreService(370): Chore service for: regionserver/f962de17fdc3:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T23:28:46,099 INFO [RS:2;f962de17fdc3:39661 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:28:46,099 INFO [regionserver/f962de17fdc3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:28:46,100 INFO [RS:2;f962de17fdc3:39661 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39661 2024-12-01T23:28:46,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f962de17fdc3,39661,1733095695603 2024-12-01T23:28:46,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:28:46,226 INFO [RS:2;f962de17fdc3:39661 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:28:46,227 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f962de17fdc3,39661,1733095695603] 2024-12-01T23:28:46,236 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f962de17fdc3,39661,1733095695603 already deleted, retry=false 2024-12-01T23:28:46,236 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f962de17fdc3,39661,1733095695603 on f962de17fdc3,36351,1733095694462 2024-12-01T23:28:46,243 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f962de17fdc3,39661,1733095695603, splitWal=true, meta=true 2024-12-01T23:28:46,247 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f962de17fdc3,39661,1733095695603, splitWal=true, meta=true 2024-12-01T23:28:46,249 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure f962de17fdc3,39661,1733095695603, splitWal=true, meta=true, isMeta: true 2024-12-01T23:28:46,251 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for f962de17fdc3,39661,1733095695603 (carryingMeta=true) f962de17fdc3,39661,1733095695603/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6233dc3d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-01T23:28:46,254 DEBUG [PEWorker-4 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting 2024-12-01T23:28:46,256 INFO [PEWorker-4 {}] master.SplitWALManager(105): f962de17fdc3,39661,1733095695603 WAL count=1, meta=true 2024-12-01T23:28:46,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta}] 2024-12-01T23:28:46,266 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:46,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta, worker=f962de17fdc3,34145,1733095695383}] 2024-12-01T23:28:46,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:28:46,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39661-0x101933997fc0003, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:28:46,338 INFO [RS:2;f962de17fdc3:39661 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:28:46,338 INFO [RS:2;f962de17fdc3:39661 {}] regionserver.HRegionServer(1031): Exiting; stopping=f962de17fdc3,39661,1733095695603; zookeeper connection closed. 2024-12-01T23:28:46,338 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@51681778 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@51681778 2024-12-01T23:28:46,401 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:28:46,402 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1] 2024-12-01T23:28:46,404 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f962de17fdc3:39661 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f962de17fdc3/172.17.0.3:39661 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:46,405 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1, error=java.net.ConnectException: Call to address=f962de17fdc3:39661 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f962de17fdc3/172.17.0.3:39661 2024-12-01T23:28:46,405 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-01T23:28:46,405 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 from cache 2024-12-01T23:28:46,405 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address f962de17fdc3:39661 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: f962de17fdc3/172.17.0.3:39661 2024-12-01T23:28:46,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34145 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-01T23:28:46,461 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta, size=7.2 K (7323bytes) 2024-12-01T23:28:46,461 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta 2024-12-01T23:28:46,462 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta after 0ms 2024-12-01T23:28:46,503 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:46,504 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta took 44ms 2024-12-01T23:28:46,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-01T23:28:46,519 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta so closing down 2024-12-01T23:28:46,519 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:46,527 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta.temp 2024-12-01T23:28:46,574 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta.temp 2024-12-01T23:28:46,575 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:46,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741892_1070 (size=7071) 2024-12-01T23:28:46,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741892_1070 (size=7071) 2024-12-01T23:28:46,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741892_1070 (size=7071) 2024-12-01T23:28:46,640 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-01T23:28:46,642 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta.temp to hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-01T23:28:46,642 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 134 ms; skipped=1; WAL=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-12-01T23:28:46,642 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta, journal: Splitting hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta, size=7.2 K (7323bytes) at 1733095726461Finishing writing output for hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta so closing down at 1733095726519 (+58 ms)Creating recovered edits writer path=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta.temp at 1733095726574 (+55 ms)3 split writer threads finished at 1733095726575 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733095726640 (+65 ms)Rename recovered edits hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta.temp to hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733095726642 (+2 ms)Processed 16 edits across 1 Regions in 134 ms; skipped=1; WAL=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1733095726642 2024-12-01T23:28:46,643 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta 2024-12-01T23:28:46,645 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-01T23:28:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-01T23:28:46,670 INFO [PEWorker-2 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta to hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/oldWALs 2024-12-01T23:28:46,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-01T23:28:46,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta, worker=f962de17fdc3,34145,1733095695383 in 401 msec 2024-12-01T23:28:46,683 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:46,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-01T23:28:46,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure f962de17fdc3%2C39661%2C1733095695603.meta.1733095701332.meta, worker=f962de17fdc3,34145,1733095695383 in 426 msec 2024-12-01T23:28:46,692 INFO [PEWorker-4 {}] master.SplitLogManager(171): hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting dir is empty, no logs to split. 2024-12-01T23:28:46,692 INFO [PEWorker-4 {}] master.SplitWALManager(105): f962de17fdc3,39661,1733095695603 WAL count=0, meta=true 2024-12-01T23:28:46,693 DEBUG [PEWorker-4 {}] procedure.ServerCrashProcedure(329): Check if f962de17fdc3,39661,1733095695603 WAL splitting is done? wals=0, meta=true 2024-12-01T23:28:46,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T23:28:46,698 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T23:28:46,700 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-01T23:28:46,850 DEBUG [f962de17fdc3:36351 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-01T23:28:46,851 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(204): Hosts are {f962de17fdc3=0} racks are {/default-rack=0} 2024-12-01T23:28:46,851 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T23:28:46,851 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T23:28:46,851 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T23:28:46,851 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T23:28:46,851 INFO [f962de17fdc3:36351 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T23:28:46,851 INFO [f962de17fdc3:36351 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T23:28:46,851 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T23:28:46,852 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:46,854 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f962de17fdc3,34145,1733095695383, state=OPENING 2024-12-01T23:28:46,911 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:28:46,913 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1] 2024-12-01T23:28:46,914 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to f962de17fdc3:39661 this server is in the failed servers list 2024-12-01T23:28:46,915 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f962de17fdc3:39661 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f962de17fdc3:39661 2024-12-01T23:28:46,915 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f962de17fdc3:39661 2024-12-01T23:28:46,915 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 from cache 2024-12-01T23:28:46,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:28:46,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:28:46,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:28:46,941 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T23:28:46,942 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:46,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f962de17fdc3,34145,1733095695383}] 2024-12-01T23:28:46,942 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:46,942 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:47,109 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T23:28:47,109 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:47,109 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T23:28:47,112 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f962de17fdc3%2C34145%2C1733095695383.meta, suffix=.meta, logDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,34145,1733095695383, archiveDir=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/oldWALs, maxLogs=32 2024-12-01T23:28:47,131 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,34145,1733095695383/f962de17fdc3%2C34145%2C1733095695383.meta.1733095727112.meta, exclude list is [], retry=0 2024-12-01T23:28:47,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:47,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:47,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:47,191 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,34145,1733095695383/f962de17fdc3%2C34145%2C1733095695383.meta.1733095727112.meta 2024-12-01T23:28:47,192 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:28:47,193 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:47,193 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T23:28:47,193 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T23:28:47,193 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T23:28:47,193 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T23:28:47,193 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:47,194 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T23:28:47,194 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T23:28:47,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T23:28:47,213 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T23:28:47,213 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:47,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:47,216 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T23:28:47,220 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T23:28:47,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:47,224 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:47,224 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T23:28:47,226 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T23:28:47,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:47,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:47,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T23:28:47,229 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T23:28:47,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:47,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T23:28:47,231 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T23:28:47,232 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740 2024-12-01T23:28:47,234 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740 2024-12-01T23:28:47,235 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-01T23:28:47,282 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:47,287 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-01T23:28:47,287 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-01T23:28:47,323 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/info/9f24ff0b58374a728cf512b2194e592f is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021./info:regioninfo/1733095725652/Put/seqid=0 2024-12-01T23:28:47,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741894_1072 (size=11177) 2024-12-01T23:28:47,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741894_1072 (size=11177) 2024-12-01T23:28:47,367 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/info/9f24ff0b58374a728cf512b2194e592f 2024-12-01T23:28:47,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741894_1072 (size=11177) 2024-12-01T23:28:47,409 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/ns/c5aaccc8023944b69cd26bf9e32f6797 is 43, key is default/ns:d/1733095701805/Put/seqid=0 2024-12-01T23:28:47,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741895_1073 (size=5153) 2024-12-01T23:28:47,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741895_1073 (size=5153) 2024-12-01T23:28:47,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741895_1073 (size=5153) 2024-12-01T23:28:47,427 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/ns/c5aaccc8023944b69cd26bf9e32f6797 2024-12-01T23:28:47,445 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/table/7ae42d2a305e4fc7bc346d9a8cabfffb is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733095722402/Put/seqid=0 2024-12-01T23:28:47,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741896_1074 (size=5431) 2024-12-01T23:28:47,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741896_1074 (size=5431) 2024-12-01T23:28:47,452 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/table/7ae42d2a305e4fc7bc346d9a8cabfffb 2024-12-01T23:28:47,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741896_1074 (size=5431) 2024-12-01T23:28:47,460 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/info/9f24ff0b58374a728cf512b2194e592f as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/info/9f24ff0b58374a728cf512b2194e592f 2024-12-01T23:28:47,467 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/info/9f24ff0b58374a728cf512b2194e592f, entries=36, sequenceid=18, filesize=10.9 K 2024-12-01T23:28:47,469 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/ns/c5aaccc8023944b69cd26bf9e32f6797 as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/ns/c5aaccc8023944b69cd26bf9e32f6797 2024-12-01T23:28:47,476 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/ns/c5aaccc8023944b69cd26bf9e32f6797, entries=2, sequenceid=18, filesize=5.0 K 2024-12-01T23:28:47,478 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/table/7ae42d2a305e4fc7bc346d9a8cabfffb as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/table/7ae42d2a305e4fc7bc346d9a8cabfffb 2024-12-01T23:28:47,485 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/table/7ae42d2a305e4fc7bc346d9a8cabfffb, entries=2, sequenceid=18, filesize=5.3 K 2024-12-01T23:28:47,486 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 199ms, sequenceid=18, compaction requested=false; wal=null 2024-12-01T23:28:47,487 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-01T23:28:47,489 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T23:28:47,489 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T23:28:47,489 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T23:28:47,491 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T23:28:47,494 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-01T23:28:47,495 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61440568, jitterRate=-0.0844641923904419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T23:28:47,495 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T23:28:47,496 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733095727194Writing region info on filesystem at 1733095727194Initializing all the Stores at 1733095727202 (+8 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733095727203 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733095727204 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095727204Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733095727204Obtaining lock to block concurrent updates at 1733095727287 (+83 ms)Preparing flush snapshotting stores in 1588230740 at 1733095727287Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733095727287Flushing stores of hbase:meta,,1.1588230740 at 1733095727287Flushing 1588230740/info: creating writer at 1733095727288 (+1 ms)Flushing 1588230740/info: appending metadata at 1733095727322 (+34 ms)Flushing 1588230740/info: closing flushed file at 1733095727322Flushing 1588230740/ns: creating writer at 1733095727391 (+69 ms)Flushing 1588230740/ns: appending metadata at 1733095727409 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733095727409Flushing 1588230740/table: creating writer at 1733095727432 (+23 ms)Flushing 1588230740/table: appending metadata at 1733095727445 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733095727445Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bd37779: reopening flushed file at 1733095727458 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13c074a9: reopening flushed file at 1733095727468 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@534349ad: reopening flushed file at 1733095727476 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 199ms, sequenceid=18, compaction requested=false; wal=null at 1733095727486 (+10 ms)Cleaning up temporary data from old regions at 1733095727489 (+3 ms)Running coprocessor post-open hooks at 1733095727495 (+6 ms)Region opened successfully at 1733095727496 (+1 ms) 2024-12-01T23:28:47,497 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733095727103 2024-12-01T23:28:47,500 DEBUG [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T23:28:47,500 INFO [RS_OPEN_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T23:28:47,501 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:47,501 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f962de17fdc3,34145,1733095695383, state=OPEN 2024-12-01T23:28:47,580 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36351 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f962de17fdc3,34145,1733095695383, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-01T23:28:47,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:28:47,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:28:47,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T23:28:47,604 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:47,604 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:47,604 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:47,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T23:28:47,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-01T23:28:47,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f962de17fdc3,34145,1733095695383 in 664 msec 2024-12-01T23:28:47,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-01T23:28:47,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 913 msec 2024-12-01T23:28:47,613 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): f962de17fdc3,39661,1733095695603 had 2 regions 2024-12-01T23:28:47,614 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f962de17fdc3,39661,1733095695603, splitWal=true, meta=true, isMeta: false 2024-12-01T23:28:47,617 INFO [PEWorker-3 {}] master.SplitWALManager(105): f962de17fdc3,39661,1733095695603 WAL count=1, meta=false 2024-12-01T23:28:47,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure f962de17fdc3%2C39661%2C1733095695603.1733095700522}] 2024-12-01T23:28:47,621 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=f962de17fdc3,33945,1733095695537 2024-12-01T23:28:47,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure f962de17fdc3%2C39661%2C1733095695603.1733095700522, worker=f962de17fdc3,33945,1733095695537}] 2024-12-01T23:28:47,776 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T23:28:47,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56023, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T23:28:47,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33945 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-01T23:28:47,801 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522, size=2.2 K (2288bytes) 2024-12-01T23:28:47,802 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522 2024-12-01T23:28:47,802 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522 after 0ms 2024-12-01T23:28:47,838 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:47,838 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522 took 37ms 2024-12-01T23:28:47,845 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 153d48efbf4361bd73ec060241e37021: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-01T23:28:47,845 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522 so closing down 2024-12-01T23:28:47,845 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:47,848 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:47,848 INFO [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-01T23:28:47,848 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522, journal: Splitting hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522, size=2.2 K (2288bytes) at 1733095727802Finishing writing output for hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522 so closing down at 1733095727845 (+43 ms)3 split writer threads finished at 1733095727848 (+3 ms)Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733095727848 2024-12-01T23:28:47,848 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522 2024-12-01T23:28:47,848 DEBUG [RS_LOG_REPLAY_OPS-regionserver/f962de17fdc3:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-01T23:28:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36351 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-01T23:28:47,857 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting/f962de17fdc3%2C39661%2C1733095695603.1733095700522 to hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/oldWALs 2024-12-01T23:28:47,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-01T23:28:47,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure f962de17fdc3%2C39661%2C1733095695603.1733095700522, worker=f962de17fdc3,33945,1733095695537 in 234 msec 2024-12-01T23:28:47,872 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=f962de17fdc3,33945,1733095695537 2024-12-01T23:28:47,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-01T23:28:47,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure f962de17fdc3%2C39661%2C1733095695603.1733095700522, worker=f962de17fdc3,33945,1733095695537 in 255 msec 2024-12-01T23:28:47,885 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/WALs/f962de17fdc3,39661,1733095695603-splitting dir is empty, no logs to split. 2024-12-01T23:28:47,885 INFO [PEWorker-5 {}] master.SplitWALManager(105): f962de17fdc3,39661,1733095695603 WAL count=0, meta=false 2024-12-01T23:28:47,885 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if f962de17fdc3,39661,1733095695603 WAL splitting is done? wals=0, meta=false 2024-12-01T23:28:47,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, ASSIGN}] 2024-12-01T23:28:47,888 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T23:28:47,891 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, ASSIGN 2024-12-01T23:28:47,897 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-01T23:28:47,921 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:28:47,925 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,34145,1733095695383, seqNum=-1] 2024-12-01T23:28:48,047 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(204): Hosts are {f962de17fdc3=0} racks are {/default-rack=0} 2024-12-01T23:28:48,047 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T23:28:48,047 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T23:28:48,047 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T23:28:48,047 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T23:28:48,047 INFO [f962de17fdc3:36351 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T23:28:48,047 INFO [f962de17fdc3:36351 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T23:28:48,047 DEBUG [f962de17fdc3:36351 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T23:28:48,048 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=OPENING, regionLocation=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:48,049 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server f962de17fdc3:39661 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f962de17fdc3/172.17.0.3:39661 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:48,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1, error=java.net.ConnectException: Call to address=f962de17fdc3:39661 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f962de17fdc3/172.17.0.3:39661 2024-12-01T23:28:48,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-01T23:28:48,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,39661,1733095695603, seqNum=-1 from cache 2024-12-01T23:28:48,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address f962de17fdc3:39661 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: f962de17fdc3/172.17.0.3:39661 2024-12-01T23:28:48,050 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to f962de17fdc3:39661 this server is in the failed servers list 2024-12-01T23:28:48,051 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f962de17fdc3:39661 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f962de17fdc3:39661 2024-12-01T23:28:48,051 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f962de17fdc3:39661 2024-12-01T23:28:48,051 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=12 from cache 2024-12-01T23:28:48,161 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T23:28:48,161 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f962de17fdc3,34145,1733095695383, seqNum=-1] 2024-12-01T23:28:48,162 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T23:28:48,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37165, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T23:28:48,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, ASSIGN because future has completed 2024-12-01T23:28:48,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,34145,1733095695383}] 2024-12-01T23:28:48,269 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=18] 2024-12-01T23:28:48,270 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to f962de17fdc3:39661 this server is in the failed servers list 2024-12-01T23:28:48,270 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=f962de17fdc3:39661 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f962de17fdc3:39661 2024-12-01T23:28:48,270 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: f962de17fdc3:39661 2024-12-01T23:28:48,270 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,39661,1733095695603, seqNum=18 from cache 2024-12-01T23:28:48,326 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:48,326 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 153d48efbf4361bd73ec060241e37021, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:48,327 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,327 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:48,327 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,327 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,329 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,330 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf1 2024-12-01T23:28:48,330 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:48,350 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf1/51b849444c1c44f0b9c6c415a8b33faf 2024-12-01T23:28:48,350 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:48,350 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,352 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 153d48efbf4361bd73ec060241e37021 columnFamilyName cf2 2024-12-01T23:28:48,352 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:48,368 DEBUG [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/cf2/0569d9a3b0a8445ca2a76e0bf45e8f45 2024-12-01T23:28:48,368 INFO [StoreOpener-153d48efbf4361bd73ec060241e37021-1 {}] regionserver.HStore(327): Store=153d48efbf4361bd73ec060241e37021/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:48,369 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,371 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,376 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,379 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,379 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,380 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T23:28:48,389 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,394 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 153d48efbf4361bd73ec060241e37021; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67540077, jitterRate=0.006425574421882629}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T23:28:48,395 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:28:48,396 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 153d48efbf4361bd73ec060241e37021: Running coprocessor pre-open hook at 1733095728327Writing region info on filesystem at 1733095728327Initializing all the Stores at 1733095728328 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095728329 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095728329Cleaning up temporary data from old regions at 1733095728379 (+50 ms)Running coprocessor post-open hooks at 1733095728395 (+16 ms)Region opened successfully at 1733095728395 2024-12-01T23:28:48,401 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., pid=21, masterSystemTime=1733095728321 2024-12-01T23:28:48,408 DEBUG [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:48,409 INFO [RS_OPEN_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:28:48,409 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=153d48efbf4361bd73ec060241e37021, regionState=OPEN, openSeqNum=18, regionLocation=f962de17fdc3,34145,1733095695383 2024-12-01T23:28:48,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,34145,1733095695383 because future has completed 2024-12-01T23:28:48,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-01T23:28:48,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 153d48efbf4361bd73ec060241e37021, server=f962de17fdc3,34145,1733095695383 in 256 msec 2024-12-01T23:28:48,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-01T23:28:48,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=153d48efbf4361bd73ec060241e37021, ASSIGN in 543 msec 2024-12-01T23:28:48,436 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server f962de17fdc3,39661,1733095695603 after splitting done 2024-12-01T23:28:48,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure f962de17fdc3,39661,1733095695603, splitWal=true, meta=true in 2.1990 sec 2024-12-01T23:28:48,586 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021., hostname=f962de17fdc3,34145,1733095695383, seqNum=18] 2024-12-01T23:28:48,615 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=402 (was 411), OpenFileDescriptor=1034 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1107 (was 1081) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3267 (was 3524) 2024-12-01T23:28:48,616 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1034 is superior to 1024 2024-12-01T23:28:48,639 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=402, OpenFileDescriptor=1034, MaxFileDescriptor=1048576, SystemLoadAverage=1107, ProcessCount=11, AvailableMemoryMB=3266 2024-12-01T23:28:48,639 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1034 is superior to 1024 2024-12-01T23:28:48,661 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:48,664 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:48,665 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:48,669 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39827849, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/hregion-39827849, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:48,687 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39827849/hregion-39827849.1733095728669, exclude list is [], retry=0 2024-12-01T23:28:48,691 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:48,692 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:48,700 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:48,731 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39827849/hregion-39827849.1733095728669 2024-12-01T23:28:48,732 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:28:48,733 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 08d13cbff909e0b4a1017712b0f5efe4, NAME => 'testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/hbase 2024-12-01T23:28:48,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741898_1076 (size=67) 2024-12-01T23:28:48,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741898_1076 (size=67) 2024-12-01T23:28:48,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741898_1076 (size=67) 2024-12-01T23:28:49,163 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:49,165 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,167 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d13cbff909e0b4a1017712b0f5efe4 columnFamilyName a 2024-12-01T23:28:49,167 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,168 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(327): Store=08d13cbff909e0b4a1017712b0f5efe4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,168 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,169 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d13cbff909e0b4a1017712b0f5efe4 columnFamilyName b 2024-12-01T23:28:49,169 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,170 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(327): Store=08d13cbff909e0b4a1017712b0f5efe4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,170 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,172 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d13cbff909e0b4a1017712b0f5efe4 columnFamilyName c 2024-12-01T23:28:49,172 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,173 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(327): Store=08d13cbff909e0b4a1017712b0f5efe4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,173 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,174 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,174 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,175 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,175 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,176 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:49,177 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,180 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:49,180 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 08d13cbff909e0b4a1017712b0f5efe4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73487874, jitterRate=0.09505465626716614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:49,182 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 08d13cbff909e0b4a1017712b0f5efe4: Writing region info on filesystem at 1733095729163Initializing all the Stores at 1733095729165 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729165Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729165Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729165Cleaning up temporary data from old regions at 1733095729175 (+10 ms)Region opened successfully at 1733095729181 (+6 ms) 2024-12-01T23:28:49,182 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 08d13cbff909e0b4a1017712b0f5efe4, disabling compactions & flushes 2024-12-01T23:28:49,182 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. 2024-12-01T23:28:49,182 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. 2024-12-01T23:28:49,182 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. after waiting 0 ms 2024-12-01T23:28:49,182 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. 2024-12-01T23:28:49,183 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. 2024-12-01T23:28:49,183 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 08d13cbff909e0b4a1017712b0f5efe4: Waiting for close lock at 1733095729182Disabling compacts and flushes for region at 1733095729182Disabling writes for close at 1733095729182Writing region close event to WAL at 1733095729183 (+1 ms)Closed at 1733095729183 2024-12-01T23:28:49,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741897_1075 (size=175) 2024-12-01T23:28:49,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741897_1075 (size=175) 2024-12-01T23:28:49,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741897_1075 (size=175) 2024-12-01T23:28:49,204 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:49,204 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39827849:(num 1733095728669) 2024-12-01T23:28:49,204 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:49,207 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:49,230 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207, exclude list is [], retry=0 2024-12-01T23:28:49,233 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:49,234 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:49,234 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:49,270 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207 2024-12-01T23:28:49,271 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:49,271 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 08d13cbff909e0b4a1017712b0f5efe4, NAME => 'testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:49,271 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:49,271 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,271 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,272 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,273 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d13cbff909e0b4a1017712b0f5efe4 columnFamilyName a 2024-12-01T23:28:49,273 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,274 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(327): Store=08d13cbff909e0b4a1017712b0f5efe4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,274 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,275 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d13cbff909e0b4a1017712b0f5efe4 columnFamilyName b 2024-12-01T23:28:49,275 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,275 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(327): Store=08d13cbff909e0b4a1017712b0f5efe4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,275 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,276 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d13cbff909e0b4a1017712b0f5efe4 columnFamilyName c 2024-12-01T23:28:49,276 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,276 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(327): Store=08d13cbff909e0b4a1017712b0f5efe4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,277 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,277 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,278 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,279 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,279 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,280 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:49,281 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,282 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 08d13cbff909e0b4a1017712b0f5efe4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60243146, jitterRate=-0.10230717062950134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:49,282 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 08d13cbff909e0b4a1017712b0f5efe4: Writing region info on filesystem at 1733095729271Initializing all the Stores at 1733095729272 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729272Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729272Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729272Cleaning up temporary data from old regions at 1733095729279 (+7 ms)Region opened successfully at 1733095729282 (+3 ms) 2024-12-01T23:28:49,307 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 08d13cbff909e0b4a1017712b0f5efe4 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-01T23:28:49,329 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/a/a3a0d237deb840ba9fa9b23180065908 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733095729282/Put/seqid=0 2024-12-01T23:28:49,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741900_1078 (size=5958) 2024-12-01T23:28:49,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741900_1078 (size=5958) 2024-12-01T23:28:49,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741900_1078 (size=5958) 2024-12-01T23:28:49,336 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/a/a3a0d237deb840ba9fa9b23180065908 2024-12-01T23:28:49,355 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/b/27370f85b53e40af9f5e6283327ae55d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733095729291/Put/seqid=0 2024-12-01T23:28:49,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741901_1079 (size=5958) 2024-12-01T23:28:49,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741901_1079 (size=5958) 2024-12-01T23:28:49,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741901_1079 (size=5958) 2024-12-01T23:28:49,367 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/b/27370f85b53e40af9f5e6283327ae55d 2024-12-01T23:28:49,412 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/c/5aeba5ea712d4d8da9bf0a71e03c887f is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733095729298/Put/seqid=0 2024-12-01T23:28:49,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741902_1080 (size=5958) 2024-12-01T23:28:49,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741902_1080 (size=5958) 2024-12-01T23:28:49,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741902_1080 (size=5958) 2024-12-01T23:28:49,435 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/c/5aeba5ea712d4d8da9bf0a71e03c887f 2024-12-01T23:28:49,455 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/a/a3a0d237deb840ba9fa9b23180065908 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/a/a3a0d237deb840ba9fa9b23180065908 2024-12-01T23:28:49,464 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/a/a3a0d237deb840ba9fa9b23180065908, entries=10, sequenceid=33, filesize=5.8 K 2024-12-01T23:28:49,466 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/b/27370f85b53e40af9f5e6283327ae55d as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/b/27370f85b53e40af9f5e6283327ae55d 2024-12-01T23:28:49,473 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/b/27370f85b53e40af9f5e6283327ae55d, entries=10, sequenceid=33, filesize=5.8 K 2024-12-01T23:28:49,474 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/c/5aeba5ea712d4d8da9bf0a71e03c887f as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/c/5aeba5ea712d4d8da9bf0a71e03c887f 2024-12-01T23:28:49,480 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/c/5aeba5ea712d4d8da9bf0a71e03c887f, entries=10, sequenceid=33, filesize=5.8 K 2024-12-01T23:28:49,482 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 08d13cbff909e0b4a1017712b0f5efe4 in 175ms, sequenceid=33, compaction requested=false 2024-12-01T23:28:49,482 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 08d13cbff909e0b4a1017712b0f5efe4: 2024-12-01T23:28:49,482 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 08d13cbff909e0b4a1017712b0f5efe4, disabling compactions & flushes 2024-12-01T23:28:49,482 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. 2024-12-01T23:28:49,482 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. 2024-12-01T23:28:49,482 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. after waiting 0 ms 2024-12-01T23:28:49,482 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. 2024-12-01T23:28:49,483 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. 2024-12-01T23:28:49,483 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 08d13cbff909e0b4a1017712b0f5efe4: Waiting for close lock at 1733095729482Disabling compacts and flushes for region at 1733095729482Disabling writes for close at 1733095729482Writing region close event to WAL at 1733095729483 (+1 ms)Closed at 1733095729483 2024-12-01T23:28:49,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741899_1077 (size=6311) 2024-12-01T23:28:49,486 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207 not finished, retry = 0 2024-12-01T23:28:49,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741899_1077 (size=6311) 2024-12-01T23:28:49,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741899_1077 (size=6311) 2024-12-01T23:28:49,592 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/b/27370f85b53e40af9f5e6283327ae55d to hdfs://localhost:35981/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/b/27370f85b53e40af9f5e6283327ae55d 2024-12-01T23:28:49,605 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207, size=6.2 K (6311bytes) 2024-12-01T23:28:49,605 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207 2024-12-01T23:28:49,606 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207 after 1ms 2024-12-01T23:28:49,636 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:49,637 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207 took 32ms 2024-12-01T23:28:49,639 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207 so closing down 2024-12-01T23:28:49,639 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:49,645 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733095729207.temp 2024-12-01T23:28:49,669 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000003-wal.1733095729207.temp 2024-12-01T23:28:49,669 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:49,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741903_1081 (size=5575) 2024-12-01T23:28:49,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741903_1081 (size=5575) 2024-12-01T23:28:49,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741903_1081 (size=5575) 2024-12-01T23:28:49,677 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000003-wal.1733095729207.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-01T23:28:49,678 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000003-wal.1733095729207.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000032 2024-12-01T23:28:49,678 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 41 ms; skipped=2; WAL=hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-01T23:28:49,679 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207, journal: Splitting hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207, size=6.2 K (6311bytes) at 1733095729605Finishing writing output for hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207 so closing down at 1733095729639 (+34 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000003-wal.1733095729207.temp at 1733095729669 (+30 ms)3 split writer threads finished at 1733095729669Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000003-wal.1733095729207.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733095729677 (+8 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000003-wal.1733095729207.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000032 at 1733095729678 (+1 ms)Processed 32 edits across 1 Regions in 41 ms; skipped=2; WAL=hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733095729678 2024-12-01T23:28:49,680 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729207 to hdfs://localhost:35981/hbase/oldWALs/wal.1733095729207 2024-12-01T23:28:49,681 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000032 2024-12-01T23:28:49,681 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:49,683 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:49,695 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729683, exclude list is [], retry=0 2024-12-01T23:28:49,697 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:49,698 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:49,698 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:49,722 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729683 2024-12-01T23:28:49,722 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:49,722 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 08d13cbff909e0b4a1017712b0f5efe4, NAME => 'testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:49,722 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:49,723 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,723 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,724 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,725 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d13cbff909e0b4a1017712b0f5efe4 columnFamilyName a 2024-12-01T23:28:49,725 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,730 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/a/a3a0d237deb840ba9fa9b23180065908 2024-12-01T23:28:49,731 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(327): Store=08d13cbff909e0b4a1017712b0f5efe4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,731 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,733 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d13cbff909e0b4a1017712b0f5efe4 columnFamilyName b 2024-12-01T23:28:49,733 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,733 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(327): Store=08d13cbff909e0b4a1017712b0f5efe4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,733 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,734 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d13cbff909e0b4a1017712b0f5efe4 columnFamilyName c 2024-12-01T23:28:49,734 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,739 DEBUG [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/c/5aeba5ea712d4d8da9bf0a71e03c887f 2024-12-01T23:28:49,739 INFO [StoreOpener-08d13cbff909e0b4a1017712b0f5efe4-1 {}] regionserver.HStore(327): Store=08d13cbff909e0b4a1017712b0f5efe4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,740 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,741 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,742 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,743 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000032 2024-12-01T23:28:49,769 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:49,770 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000032 2024-12-01T23:28:49,770 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 08d13cbff909e0b4a1017712b0f5efe4 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-01T23:28:49,787 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/b/1995a3fbd2cd482f810a578940c90cd6 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733095729291/Put/seqid=0 2024-12-01T23:28:49,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741905_1083 (size=5958) 2024-12-01T23:28:49,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741905_1083 (size=5958) 2024-12-01T23:28:49,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741905_1083 (size=5958) 2024-12-01T23:28:49,800 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/b/1995a3fbd2cd482f810a578940c90cd6 2024-12-01T23:28:49,806 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/.tmp/b/1995a3fbd2cd482f810a578940c90cd6 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/b/1995a3fbd2cd482f810a578940c90cd6 2024-12-01T23:28:49,813 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/b/1995a3fbd2cd482f810a578940c90cd6, entries=10, sequenceid=32, filesize=5.8 K 2024-12-01T23:28:49,813 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 08d13cbff909e0b4a1017712b0f5efe4 in 43ms, sequenceid=32, compaction requested=false; wal=null 2024-12-01T23:28:49,814 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/0000000000000000032 2024-12-01T23:28:49,815 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,815 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,816 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:49,818 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 08d13cbff909e0b4a1017712b0f5efe4 2024-12-01T23:28:49,820 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/08d13cbff909e0b4a1017712b0f5efe4/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-01T23:28:49,821 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 08d13cbff909e0b4a1017712b0f5efe4; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68557613, jitterRate=0.021588042378425598}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:49,822 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 08d13cbff909e0b4a1017712b0f5efe4: Writing region info on filesystem at 1733095729723Initializing all the Stores at 1733095729724 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729724Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729724Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729724Obtaining lock to block concurrent updates at 1733095729770 (+46 ms)Preparing flush snapshotting stores in 08d13cbff909e0b4a1017712b0f5efe4 at 1733095729770Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733095729770Flushing stores of testReplayEditsWrittenViaHRegion,,1733095728662.08d13cbff909e0b4a1017712b0f5efe4. at 1733095729770Flushing 08d13cbff909e0b4a1017712b0f5efe4/b: creating writer at 1733095729771 (+1 ms)Flushing 08d13cbff909e0b4a1017712b0f5efe4/b: appending metadata at 1733095729786 (+15 ms)Flushing 08d13cbff909e0b4a1017712b0f5efe4/b: closing flushed file at 1733095729787 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6291d859: reopening flushed file at 1733095729805 (+18 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 08d13cbff909e0b4a1017712b0f5efe4 in 43ms, sequenceid=32, compaction requested=false; wal=null at 1733095729813 (+8 ms)Cleaning up temporary data from old regions at 1733095729815 (+2 ms)Region opened successfully at 1733095729822 (+7 ms) 2024-12-01T23:28:49,847 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=411 (was 402) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:48038 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:35812 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:50470 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:51580 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:59234 [Waiting for operation #50] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:36800 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1110 (was 1034) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1098 (was 1107), ProcessCount=11 (was 11), AvailableMemoryMB=3243 (was 3266) 2024-12-01T23:28:49,847 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1110 is superior to 1024 2024-12-01T23:28:49,863 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=411, OpenFileDescriptor=1110, MaxFileDescriptor=1048576, SystemLoadAverage=1098, ProcessCount=11, AvailableMemoryMB=3243 2024-12-01T23:28:49,863 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1110 is superior to 1024 2024-12-01T23:28:49,877 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:49,879 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:49,879 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:49,882 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-48512635, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/hregion-48512635, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:49,893 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-48512635/hregion-48512635.1733095729882, exclude list is [], retry=0 2024-12-01T23:28:49,896 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:49,896 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:49,896 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:49,921 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-48512635/hregion-48512635.1733095729882 2024-12-01T23:28:49,921 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:28:49,922 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 577494c3125fc7281495c3be8db9d358, NAME => 'testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/hbase 2024-12-01T23:28:49,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741907_1085 (size=68) 2024-12-01T23:28:49,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741907_1085 (size=68) 2024-12-01T23:28:49,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741907_1085 (size=68) 2024-12-01T23:28:49,931 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:49,931 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:49,933 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 577494c3125fc7281495c3be8db9d358 columnFamilyName a 2024-12-01T23:28:49,933 DEBUG [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,934 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(327): Store=577494c3125fc7281495c3be8db9d358/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,934 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:49,935 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 577494c3125fc7281495c3be8db9d358 columnFamilyName b 2024-12-01T23:28:49,935 DEBUG [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,936 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(327): Store=577494c3125fc7281495c3be8db9d358/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,936 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:49,938 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 577494c3125fc7281495c3be8db9d358 columnFamilyName c 2024-12-01T23:28:49,938 DEBUG [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:49,938 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(327): Store=577494c3125fc7281495c3be8db9d358/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:49,939 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:49,939 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:49,940 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:49,941 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:49,941 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:49,941 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:49,942 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:49,944 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:49,945 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 577494c3125fc7281495c3be8db9d358; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72184421, jitterRate=0.07563169300556183}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:49,945 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 577494c3125fc7281495c3be8db9d358: Writing region info on filesystem at 1733095729931Initializing all the Stores at 1733095729931Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729931Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729931Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095729931Cleaning up temporary data from old regions at 1733095729941 (+10 ms)Region opened successfully at 1733095729945 (+4 ms) 2024-12-01T23:28:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 577494c3125fc7281495c3be8db9d358, disabling compactions & flushes 2024-12-01T23:28:49,946 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. 2024-12-01T23:28:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. 2024-12-01T23:28:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. after waiting 0 ms 2024-12-01T23:28:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. 2024-12-01T23:28:49,946 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. 2024-12-01T23:28:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 577494c3125fc7281495c3be8db9d358: Waiting for close lock at 1733095729946Disabling compacts and flushes for region at 1733095729946Disabling writes for close at 1733095729946Writing region close event to WAL at 1733095729946Closed at 1733095729946 2024-12-01T23:28:49,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741906_1084 (size=175) 2024-12-01T23:28:49,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741906_1084 (size=175) 2024-12-01T23:28:49,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741906_1084 (size=175) 2024-12-01T23:28:49,951 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:49,951 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-48512635:(num 1733095729882) 2024-12-01T23:28:49,951 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:49,953 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:49,965 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954, exclude list is [], retry=0 2024-12-01T23:28:49,969 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:49,969 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:49,969 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:50,004 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954 2024-12-01T23:28:50,004 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:28:50,079 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 577494c3125fc7281495c3be8db9d358, NAME => 'testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:50,082 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,082 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:50,082 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,082 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,089 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,090 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 577494c3125fc7281495c3be8db9d358 columnFamilyName a 2024-12-01T23:28:50,090 DEBUG [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:50,091 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(327): Store=577494c3125fc7281495c3be8db9d358/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:50,092 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,093 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 577494c3125fc7281495c3be8db9d358 columnFamilyName b 2024-12-01T23:28:50,094 DEBUG [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:50,094 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(327): Store=577494c3125fc7281495c3be8db9d358/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:50,095 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,096 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 577494c3125fc7281495c3be8db9d358 columnFamilyName c 2024-12-01T23:28:50,096 DEBUG [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:50,096 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(327): Store=577494c3125fc7281495c3be8db9d358/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:50,097 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,098 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,099 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,100 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,100 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,101 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:50,103 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,105 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 577494c3125fc7281495c3be8db9d358; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59108087, jitterRate=-0.11922086775302887}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:50,105 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:50,106 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 577494c3125fc7281495c3be8db9d358: Running coprocessor pre-open hook at 1733095730082Writing region info on filesystem at 1733095730082Initializing all the Stores at 1733095730085 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095730085Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095730088 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095730088Cleaning up temporary data from old regions at 1733095730100 (+12 ms)Running coprocessor post-open hooks at 1733095730105 (+5 ms)Region opened successfully at 1733095730106 (+1 ms) 2024-12-01T23:28:50,154 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 577494c3125fc7281495c3be8db9d358 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-01T23:28:50,160 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 577494c3125fc7281495c3be8db9d358/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:51,161 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 577494c3125fc7281495c3be8db9d358/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:52,162 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 577494c3125fc7281495c3be8db9d358/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:53,163 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 577494c3125fc7281495c3be8db9d358/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:53,830 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T23:28:54,163 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 577494c3125fc7281495c3be8db9d358/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:54,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-01T23:28:54,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-01T23:28:54,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-01T23:28:54,894 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-01T23:28:55,164 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 577494c3125fc7281495c3be8db9d358/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:56,165 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 577494c3125fc7281495c3be8db9d358/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:57,166 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 577494c3125fc7281495c3be8db9d358/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:58,167 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 577494c3125fc7281495c3be8db9d358/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:59,168 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 577494c3125fc7281495c3be8db9d358/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:28:59,169 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 577494c3125fc7281495c3be8db9d358: 2024-12-01T23:28:59,169 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. 2024-12-01T23:28:59,185 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 577494c3125fc7281495c3be8db9d358: 2024-12-01T23:28:59,185 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-01T23:28:59,185 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 577494c3125fc7281495c3be8db9d358, disabling compactions & flushes 2024-12-01T23:28:59,185 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. 2024-12-01T23:28:59,185 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. 2024-12-01T23:28:59,185 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. after waiting 0 ms 2024-12-01T23:28:59,185 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. 2024-12-01T23:28:59,186 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. 2024-12-01T23:28:59,186 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. 2024-12-01T23:28:59,186 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 577494c3125fc7281495c3be8db9d358: Waiting for close lock at 1733095739185Running coprocessor pre-close hooks at 1733095739185Disabling compacts and flushes for region at 1733095739185Disabling writes for close at 1733095739185Writing region close event to WAL at 1733095739186 (+1 ms)Running coprocessor post-close hooks at 1733095739186Closed at 1733095739186 2024-12-01T23:28:59,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741908_1086 (size=4236) 2024-12-01T23:28:59,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741908_1086 (size=4236) 2024-12-01T23:28:59,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741908_1086 (size=4236) 2024-12-01T23:28:59,215 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954, size=4.1 K (4236bytes) 2024-12-01T23:28:59,215 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954 2024-12-01T23:28:59,215 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954 after 0ms 2024-12-01T23:28:59,240 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:59,240 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954 took 26ms 2024-12-01T23:28:59,243 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954 so closing down 2024-12-01T23:28:59,243 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:59,244 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733095729954.temp 2024-12-01T23:28:59,269 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000004-wal.1733095729954.temp 2024-12-01T23:28:59,269 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:59,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741909_1087 (size=3245) 2024-12-01T23:28:59,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741909_1087 (size=3245) 2024-12-01T23:28:59,287 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000004-wal.1733095729954.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-01T23:28:59,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741909_1087 (size=3245) 2024-12-01T23:28:59,289 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000004-wal.1733095729954.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000026 2024-12-01T23:28:59,289 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 48 ms; skipped=3; WAL=hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-01T23:28:59,289 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954, journal: Splitting hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954, size=4.1 K (4236bytes) at 1733095739215Finishing writing output for hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954 so closing down at 1733095739243 (+28 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000004-wal.1733095729954.temp at 1733095739269 (+26 ms)3 split writer threads finished at 1733095739269Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000004-wal.1733095729954.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733095739287 (+18 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000004-wal.1733095729954.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000026 at 1733095739289 (+2 ms)Processed 23 edits across 1 Regions in 48 ms; skipped=3; WAL=hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733095739289 2024-12-01T23:28:59,292 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095729954 to hdfs://localhost:35981/hbase/oldWALs/wal.1733095729954 2024-12-01T23:28:59,294 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000026 2024-12-01T23:28:59,294 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:59,297 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:59,310 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095739298, exclude list is [], retry=0 2024-12-01T23:28:59,313 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:59,313 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:59,313 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:59,352 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095739298 2024-12-01T23:28:59,352 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:28:59,353 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 577494c3125fc7281495c3be8db9d358, NAME => 'testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:59,353 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,354 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:59,354 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,354 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,356 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,357 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 577494c3125fc7281495c3be8db9d358 columnFamilyName a 2024-12-01T23:28:59,357 DEBUG [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:59,358 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(327): Store=577494c3125fc7281495c3be8db9d358/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:59,358 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,359 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 577494c3125fc7281495c3be8db9d358 columnFamilyName b 2024-12-01T23:28:59,359 DEBUG [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:59,360 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(327): Store=577494c3125fc7281495c3be8db9d358/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:59,360 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,361 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 577494c3125fc7281495c3be8db9d358 columnFamilyName c 2024-12-01T23:28:59,361 DEBUG [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:59,362 INFO [StoreOpener-577494c3125fc7281495c3be8db9d358-1 {}] regionserver.HStore(327): Store=577494c3125fc7281495c3be8db9d358/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:59,362 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,363 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,365 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,366 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000026 2024-12-01T23:28:59,400 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:59,402 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000026 2024-12-01T23:28:59,402 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 577494c3125fc7281495c3be8db9d358 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-01T23:28:59,425 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/.tmp/a/b970ab8f2b22433c9cb561b4f2385085 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733095739173/Put/seqid=0 2024-12-01T23:28:59,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741911_1089 (size=5523) 2024-12-01T23:28:59,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741911_1089 (size=5523) 2024-12-01T23:28:59,454 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/.tmp/a/b970ab8f2b22433c9cb561b4f2385085 2024-12-01T23:28:59,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741911_1089 (size=5523) 2024-12-01T23:28:59,483 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/.tmp/b/1bac02399aff46d18e7e0338ccfc1f24 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733095739169/Put/seqid=0 2024-12-01T23:28:59,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741912_1090 (size=5524) 2024-12-01T23:28:59,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741912_1090 (size=5524) 2024-12-01T23:28:59,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741912_1090 (size=5524) 2024-12-01T23:28:59,508 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/.tmp/b/1bac02399aff46d18e7e0338ccfc1f24 2024-12-01T23:28:59,532 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/.tmp/c/02f744610e5344b2a94d7f8975f36ee0 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733095739170/Put/seqid=0 2024-12-01T23:28:59,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741913_1091 (size=5457) 2024-12-01T23:28:59,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741913_1091 (size=5457) 2024-12-01T23:28:59,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741913_1091 (size=5457) 2024-12-01T23:28:59,556 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/.tmp/c/02f744610e5344b2a94d7f8975f36ee0 2024-12-01T23:28:59,564 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/.tmp/a/b970ab8f2b22433c9cb561b4f2385085 as hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/a/b970ab8f2b22433c9cb561b4f2385085 2024-12-01T23:28:59,572 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/a/b970ab8f2b22433c9cb561b4f2385085, entries=7, sequenceid=26, filesize=5.4 K 2024-12-01T23:28:59,573 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/.tmp/b/1bac02399aff46d18e7e0338ccfc1f24 as hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/b/1bac02399aff46d18e7e0338ccfc1f24 2024-12-01T23:28:59,580 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/b/1bac02399aff46d18e7e0338ccfc1f24, entries=7, sequenceid=26, filesize=5.4 K 2024-12-01T23:28:59,581 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/.tmp/c/02f744610e5344b2a94d7f8975f36ee0 as hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/c/02f744610e5344b2a94d7f8975f36ee0 2024-12-01T23:28:59,589 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/c/02f744610e5344b2a94d7f8975f36ee0, entries=6, sequenceid=26, filesize=5.3 K 2024-12-01T23:28:59,590 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 577494c3125fc7281495c3be8db9d358 in 187ms, sequenceid=26, compaction requested=false; wal=null 2024-12-01T23:28:59,591 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/0000000000000000026 2024-12-01T23:28:59,592 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,592 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,594 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:28:59,596 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,598 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsAfterAbortingFlush/577494c3125fc7281495c3be8db9d358/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-01T23:28:59,600 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 577494c3125fc7281495c3be8db9d358; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62453987, jitterRate=-0.069363072514534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:28:59,600 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 577494c3125fc7281495c3be8db9d358 2024-12-01T23:28:59,601 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 577494c3125fc7281495c3be8db9d358: Running coprocessor pre-open hook at 1733095739354Writing region info on filesystem at 1733095739354Initializing all the Stores at 1733095739355 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095739355Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095739355Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095739355Obtaining lock to block concurrent updates at 1733095739402 (+47 ms)Preparing flush snapshotting stores in 577494c3125fc7281495c3be8db9d358 at 1733095739402Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733095739402Flushing stores of testReplayEditsAfterAbortingFlush,,1733095729878.577494c3125fc7281495c3be8db9d358. at 1733095739402Flushing 577494c3125fc7281495c3be8db9d358/a: creating writer at 1733095739402Flushing 577494c3125fc7281495c3be8db9d358/a: appending metadata at 1733095739424 (+22 ms)Flushing 577494c3125fc7281495c3be8db9d358/a: closing flushed file at 1733095739424Flushing 577494c3125fc7281495c3be8db9d358/b: creating writer at 1733095739461 (+37 ms)Flushing 577494c3125fc7281495c3be8db9d358/b: appending metadata at 1733095739482 (+21 ms)Flushing 577494c3125fc7281495c3be8db9d358/b: closing flushed file at 1733095739483 (+1 ms)Flushing 577494c3125fc7281495c3be8db9d358/c: creating writer at 1733095739514 (+31 ms)Flushing 577494c3125fc7281495c3be8db9d358/c: appending metadata at 1733095739532 (+18 ms)Flushing 577494c3125fc7281495c3be8db9d358/c: closing flushed file at 1733095739532Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2baf518f: reopening flushed file at 1733095739563 (+31 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35876dd8: reopening flushed file at 1733095739572 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ba19289: reopening flushed file at 1733095739580 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 577494c3125fc7281495c3be8db9d358 in 187ms, sequenceid=26, compaction requested=false; wal=null at 1733095739590 (+10 ms)Cleaning up temporary data from old regions at 1733095739592 (+2 ms)Running coprocessor post-open hooks at 1733095739600 (+8 ms)Region opened successfully at 1733095739601 (+1 ms) 2024-12-01T23:28:59,628 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=416 (was 411) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:58418 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:35708 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:39396 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:39374 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:58464 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:35700 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1184 (was 1110) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1064 (was 1098), ProcessCount=11 (was 11), AvailableMemoryMB=3400 (was 3243) - AvailableMemoryMB LEAK? - 2024-12-01T23:28:59,628 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-12-01T23:28:59,641 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=416, OpenFileDescriptor=1184, MaxFileDescriptor=1048576, SystemLoadAverage=1064, ProcessCount=11, AvailableMemoryMB=3397 2024-12-01T23:28:59,641 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-12-01T23:28:59,655 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:59,657 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:28:59,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:28:59,661 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-13335580, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/hregion-13335580, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:59,674 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-13335580/hregion-13335580.1733095739661, exclude list is [], retry=0 2024-12-01T23:28:59,678 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:59,678 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:59,679 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:59,713 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-13335580/hregion-13335580.1733095739661 2024-12-01T23:28:59,713 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:59,714 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 00533637de1bb83368d706186cd3c77b, NAME => 'testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/hbase 2024-12-01T23:28:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741915_1093 (size=61) 2024-12-01T23:28:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741915_1093 (size=61) 2024-12-01T23:28:59,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741915_1093 (size=61) 2024-12-01T23:28:59,728 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:59,732 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,734 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00533637de1bb83368d706186cd3c77b columnFamilyName a 2024-12-01T23:28:59,735 DEBUG [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:59,735 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] regionserver.HStore(327): Store=00533637de1bb83368d706186cd3c77b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:59,736 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,737 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,737 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,738 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,738 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,740 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,743 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:28:59,743 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 00533637de1bb83368d706186cd3c77b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73459864, jitterRate=0.09463727474212646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:28:59,744 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 00533637de1bb83368d706186cd3c77b: Writing region info on filesystem at 1733095739728Initializing all the Stores at 1733095739729 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095739730 (+1 ms)Cleaning up temporary data from old regions at 1733095739738 (+8 ms)Region opened successfully at 1733095739744 (+6 ms) 2024-12-01T23:28:59,744 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 00533637de1bb83368d706186cd3c77b, disabling compactions & flushes 2024-12-01T23:28:59,744 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. 2024-12-01T23:28:59,744 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. 2024-12-01T23:28:59,744 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. after waiting 0 ms 2024-12-01T23:28:59,744 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. 2024-12-01T23:28:59,745 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. 2024-12-01T23:28:59,745 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 00533637de1bb83368d706186cd3c77b: Waiting for close lock at 1733095739744Disabling compacts and flushes for region at 1733095739744Disabling writes for close at 1733095739744Writing region close event to WAL at 1733095739745 (+1 ms)Closed at 1733095739745 2024-12-01T23:28:59,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741914_1092 (size=175) 2024-12-01T23:28:59,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741914_1092 (size=175) 2024-12-01T23:28:59,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741914_1092 (size=175) 2024-12-01T23:28:59,751 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:28:59,751 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-13335580:(num 1733095739661) 2024-12-01T23:28:59,751 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:28:59,753 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:28:59,770 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754, exclude list is [], retry=0 2024-12-01T23:28:59,773 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:28:59,773 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:28:59,774 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:28:59,823 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754 2024-12-01T23:28:59,824 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:28:59,824 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 00533637de1bb83368d706186cd3c77b, NAME => 'testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:28:59,824 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:28:59,824 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,824 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,826 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,827 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00533637de1bb83368d706186cd3c77b columnFamilyName a 2024-12-01T23:28:59,827 DEBUG [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:28:59,827 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] regionserver.HStore(327): Store=00533637de1bb83368d706186cd3c77b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:28:59,827 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,828 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,829 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,829 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,829 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,831 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:28:59,832 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 00533637de1bb83368d706186cd3c77b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60699556, jitterRate=-0.09550613164901733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:28:59,833 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 00533637de1bb83368d706186cd3c77b: Writing region info on filesystem at 1733095739824Initializing all the Stores at 1733095739825 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095739825Cleaning up temporary data from old regions at 1733095739829 (+4 ms)Region opened successfully at 1733095739833 (+4 ms) 2024-12-01T23:28:59,841 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 00533637de1bb83368d706186cd3c77b, disabling compactions & flushes 2024-12-01T23:28:59,841 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. 2024-12-01T23:28:59,842 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. 2024-12-01T23:28:59,842 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. after waiting 0 ms 2024-12-01T23:28:59,842 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. 2024-12-01T23:28:59,842 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. 2024-12-01T23:28:59,842 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. 2024-12-01T23:28:59,842 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 00533637de1bb83368d706186cd3c77b: Waiting for close lock at 1733095739841Disabling compacts and flushes for region at 1733095739841Disabling writes for close at 1733095739842 (+1 ms)Writing region close event to WAL at 1733095739842Closed at 1733095739842 2024-12-01T23:28:59,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741916_1094 (size=1795) 2024-12-01T23:28:59,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741916_1094 (size=1795) 2024-12-01T23:28:59,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741916_1094 (size=1795) 2024-12-01T23:28:59,865 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754, size=1.8 K (1795bytes) 2024-12-01T23:28:59,865 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754 2024-12-01T23:28:59,866 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754 after 1ms 2024-12-01T23:28:59,893 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:28:59,894 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754 took 28ms 2024-12-01T23:28:59,895 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754 so closing down 2024-12-01T23:28:59,896 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:28:59,897 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733095739754.temp 2024-12-01T23:28:59,929 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000003-wal.1733095739754.temp 2024-12-01T23:28:59,929 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:28:59,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741917_1095 (size=1795) 2024-12-01T23:28:59,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741917_1095 (size=1795) 2024-12-01T23:28:59,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741917_1095 (size=1795) 2024-12-01T23:28:59,936 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000003-wal.1733095739754.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-01T23:28:59,937 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000003-wal.1733095739754.temp to hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000012 2024-12-01T23:28:59,937 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-01T23:28:59,937 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754, journal: Splitting hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754, size=1.8 K (1795bytes) at 1733095739865Finishing writing output for hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754 so closing down at 1733095739895 (+30 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000003-wal.1733095739754.temp at 1733095739929 (+34 ms)3 split writer threads finished at 1733095739929Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000003-wal.1733095739754.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733095739936 (+7 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000003-wal.1733095739754.temp to hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000012 at 1733095739937 (+1 ms)Processed 10 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733095739937 2024-12-01T23:28:59,938 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095739754 to hdfs://localhost:35981/hbase/oldWALs/wal.1733095739754 2024-12-01T23:28:59,939 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000012 2024-12-01T23:28:59,971 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:29:00,430 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:29:00,437 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:29:00,455 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095740437, exclude list is [], retry=0 2024-12-01T23:29:00,461 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:29:00,461 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:29:00,461 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:29:00,504 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095740437 2024-12-01T23:29:00,504 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:29:00,504 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 00533637de1bb83368d706186cd3c77b, NAME => 'testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:29:00,505 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:29:00,505 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:00,505 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:00,509 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:00,512 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00533637de1bb83368d706186cd3c77b columnFamilyName a 2024-12-01T23:29:00,512 DEBUG [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:00,514 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] regionserver.HStore(327): Store=00533637de1bb83368d706186cd3c77b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:00,514 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:00,516 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:00,517 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:00,518 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000012 2024-12-01T23:29:00,557 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:29:00,558 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000012 2024-12-01T23:29:00,558 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 00533637de1bb83368d706186cd3c77b 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-01T23:29:00,585 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/.tmp/a/908b33edef944c9fba9294bfa81dfe4c is 79, key is testDatalossWhenInputError/a:x0/1733095739833/Put/seqid=0 2024-12-01T23:29:00,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741919_1097 (size=5808) 2024-12-01T23:29:00,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741919_1097 (size=5808) 2024-12-01T23:29:00,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741919_1097 (size=5808) 2024-12-01T23:29:01,017 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/.tmp/a/908b33edef944c9fba9294bfa81dfe4c 2024-12-01T23:29:01,026 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/.tmp/a/908b33edef944c9fba9294bfa81dfe4c as hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/a/908b33edef944c9fba9294bfa81dfe4c 2024-12-01T23:29:01,034 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/a/908b33edef944c9fba9294bfa81dfe4c, entries=10, sequenceid=12, filesize=5.7 K 2024-12-01T23:29:01,034 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 00533637de1bb83368d706186cd3c77b in 476ms, sequenceid=12, compaction requested=false; wal=null 2024-12-01T23:29:01,035 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/0000000000000000012 2024-12-01T23:29:01,036 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,036 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,039 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,041 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-01T23:29:01,042 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 00533637de1bb83368d706186cd3c77b; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60964121, jitterRate=-0.09156380593776703}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:29:01,042 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 00533637de1bb83368d706186cd3c77b: Writing region info on filesystem at 1733095740505Initializing all the Stores at 1733095740509 (+4 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095740509Obtaining lock to block concurrent updates at 1733095740558 (+49 ms)Preparing flush snapshotting stores in 00533637de1bb83368d706186cd3c77b at 1733095740558Finished memstore snapshotting testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733095740558Flushing stores of testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b. at 1733095740558Flushing 00533637de1bb83368d706186cd3c77b/a: creating writer at 1733095740558Flushing 00533637de1bb83368d706186cd3c77b/a: appending metadata at 1733095740584 (+26 ms)Flushing 00533637de1bb83368d706186cd3c77b/a: closing flushed file at 1733095740584Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56c49647: reopening flushed file at 1733095741024 (+440 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 00533637de1bb83368d706186cd3c77b in 476ms, sequenceid=12, compaction requested=false; wal=null at 1733095741034 (+10 ms)Cleaning up temporary data from old regions at 1733095741036 (+2 ms)Region opened successfully at 1733095741042 (+6 ms) 2024-12-01T23:29:01,045 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 00533637de1bb83368d706186cd3c77b, NAME => 'testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:29:01,045 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733095739656.00533637de1bb83368d706186cd3c77b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:29:01,045 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,045 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,047 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,047 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00533637de1bb83368d706186cd3c77b columnFamilyName a 2024-12-01T23:29:01,047 DEBUG [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:01,052 DEBUG [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/a/908b33edef944c9fba9294bfa81dfe4c 2024-12-01T23:29:01,052 INFO [StoreOpener-00533637de1bb83368d706186cd3c77b-1 {}] regionserver.HStore(327): Store=00533637de1bb83368d706186cd3c77b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:01,052 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,053 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,054 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,054 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,054 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,056 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 00533637de1bb83368d706186cd3c77b 2024-12-01T23:29:01,057 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testDatalossWhenInputError/00533637de1bb83368d706186cd3c77b/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-01T23:29:01,058 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 00533637de1bb83368d706186cd3c77b; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70839574, jitterRate=0.05559191107749939}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T23:29:01,058 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 00533637de1bb83368d706186cd3c77b: Writing region info on filesystem at 1733095741045Initializing all the Stores at 1733095741046 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095741046Cleaning up temporary data from old regions at 1733095741054 (+8 ms)Region opened successfully at 1733095741058 (+4 ms) 2024-12-01T23:29:01,073 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=424 (was 416) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:39448 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:35788 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:58498 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-959958609-172.17.0.3-1733095688324:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:39396 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:58464 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:35700 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1184) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1064 (was 1064), ProcessCount=11 (was 11), AvailableMemoryMB=3461 (was 3397) - AvailableMemoryMB LEAK? - 2024-12-01T23:29:01,073 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-01T23:29:01,085 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=424, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=1064, ProcessCount=11, AvailableMemoryMB=3460 2024-12-01T23:29:01,085 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-01T23:29:01,101 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:29:01,102 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:29:01,103 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:29:01,105 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11045476, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/hregion-11045476, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:29:01,119 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11045476/hregion-11045476.1733095741105, exclude list is [], retry=0 2024-12-01T23:29:01,121 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:29:01,121 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:29:01,121 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:29:01,146 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11045476/hregion-11045476.1733095741105 2024-12-01T23:29:01,146 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:29:01,146 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7f5f4dca9fa6f9bee3b33a43f8499778, NAME => 'testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/hbase 2024-12-01T23:29:01,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741921_1099 (size=63) 2024-12-01T23:29:01,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741921_1099 (size=63) 2024-12-01T23:29:01,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741921_1099 (size=63) 2024-12-01T23:29:01,154 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:29:01,155 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,156 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5f4dca9fa6f9bee3b33a43f8499778 columnFamilyName a 2024-12-01T23:29:01,157 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:01,157 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(327): Store=7f5f4dca9fa6f9bee3b33a43f8499778/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:01,157 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,158 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5f4dca9fa6f9bee3b33a43f8499778 columnFamilyName b 2024-12-01T23:29:01,158 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:01,159 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(327): Store=7f5f4dca9fa6f9bee3b33a43f8499778/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:01,159 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,160 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5f4dca9fa6f9bee3b33a43f8499778 columnFamilyName c 2024-12-01T23:29:01,160 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:01,161 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(327): Store=7f5f4dca9fa6f9bee3b33a43f8499778/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:01,161 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,162 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,162 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,163 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,163 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,163 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:29:01,165 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,167 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:29:01,168 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7f5f4dca9fa6f9bee3b33a43f8499778; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73829730, jitterRate=0.10014870762825012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:29:01,168 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7f5f4dca9fa6f9bee3b33a43f8499778: Writing region info on filesystem at 1733095741154Initializing all the Stores at 1733095741155 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095741155Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095741155Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095741155Cleaning up temporary data from old regions at 1733095741163 (+8 ms)Region opened successfully at 1733095741168 (+5 ms) 2024-12-01T23:29:01,169 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7f5f4dca9fa6f9bee3b33a43f8499778, disabling compactions & flushes 2024-12-01T23:29:01,169 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. 2024-12-01T23:29:01,169 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. 2024-12-01T23:29:01,169 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. after waiting 0 ms 2024-12-01T23:29:01,169 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. 2024-12-01T23:29:01,169 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. 2024-12-01T23:29:01,169 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7f5f4dca9fa6f9bee3b33a43f8499778: Waiting for close lock at 1733095741168Disabling compacts and flushes for region at 1733095741168Disabling writes for close at 1733095741169 (+1 ms)Writing region close event to WAL at 1733095741169Closed at 1733095741169 2024-12-01T23:29:01,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741920_1098 (size=175) 2024-12-01T23:29:01,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741920_1098 (size=175) 2024-12-01T23:29:01,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741920_1098 (size=175) 2024-12-01T23:29:01,176 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:29:01,176 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11045476:(num 1733095741105) 2024-12-01T23:29:01,176 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:29:01,177 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:29:01,189 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178, exclude list is [], retry=0 2024-12-01T23:29:01,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:29:01,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:29:01,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:29:01,220 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 2024-12-01T23:29:01,221 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:29:01,221 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f5f4dca9fa6f9bee3b33a43f8499778, NAME => 'testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:29:01,221 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:29:01,221 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,222 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,223 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,224 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5f4dca9fa6f9bee3b33a43f8499778 columnFamilyName a 2024-12-01T23:29:01,224 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:01,225 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(327): Store=7f5f4dca9fa6f9bee3b33a43f8499778/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:01,225 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,226 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5f4dca9fa6f9bee3b33a43f8499778 columnFamilyName b 2024-12-01T23:29:01,226 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:01,226 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(327): Store=7f5f4dca9fa6f9bee3b33a43f8499778/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:01,226 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,227 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5f4dca9fa6f9bee3b33a43f8499778 columnFamilyName c 2024-12-01T23:29:01,227 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:01,227 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(327): Store=7f5f4dca9fa6f9bee3b33a43f8499778/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:01,228 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,228 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,230 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,231 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,231 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,231 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:29:01,233 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:01,233 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7f5f4dca9fa6f9bee3b33a43f8499778; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60834710, jitterRate=-0.09349218010902405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:29:01,234 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7f5f4dca9fa6f9bee3b33a43f8499778: Writing region info on filesystem at 1733095741222Initializing all the Stores at 1733095741223 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095741223Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095741223Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095741223Cleaning up temporary data from old regions at 1733095741231 (+8 ms)Region opened successfully at 1733095741234 (+3 ms) 2024-12-01T23:29:01,238 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733095741238/Put/seqid=0 2024-12-01T23:29:01,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741923_1101 (size=4875) 2024-12-01T23:29:01,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741923_1101 (size=4875) 2024-12-01T23:29:01,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741923_1101 (size=4875) 2024-12-01T23:29:01,247 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733095741246/Put/seqid=0 2024-12-01T23:29:01,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741924_1102 (size=4875) 2024-12-01T23:29:01,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741924_1102 (size=4875) 2024-12-01T23:29:01,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741924_1102 (size=4875) 2024-12-01T23:29:01,259 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733095741258/Put/seqid=0 2024-12-01T23:29:01,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741925_1103 (size=4875) 2024-12-01T23:29:01,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741925_1103 (size=4875) 2024-12-01T23:29:01,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741925_1103 (size=4875) 2024-12-01T23:29:01,266 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 7f5f4dca9fa6f9bee3b33a43f8499778/a 2024-12-01T23:29:01,270 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-01T23:29:01,270 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T23:29:01,270 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 7f5f4dca9fa6f9bee3b33a43f8499778/a 2024-12-01T23:29:01,274 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-01T23:29:01,274 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T23:29:01,274 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 7f5f4dca9fa6f9bee3b33a43f8499778/a 2024-12-01T23:29:01,278 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-01T23:29:01,278 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T23:29:01,278 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7f5f4dca9fa6f9bee3b33a43f8499778 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-01T23:29:01,293 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/.tmp/a/9f6d020a52a74031b025d8a8f7436be3 is 55, key is testCompactedBulkLoadedFiles/a:a/1733095741234/Put/seqid=0 2024-12-01T23:29:01,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741926_1104 (size=5107) 2024-12-01T23:29:01,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741926_1104 (size=5107) 2024-12-01T23:29:01,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741926_1104 (size=5107) 2024-12-01T23:29:01,303 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/.tmp/a/9f6d020a52a74031b025d8a8f7436be3 2024-12-01T23:29:01,311 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/.tmp/a/9f6d020a52a74031b025d8a8f7436be3 as hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/9f6d020a52a74031b025d8a8f7436be3 2024-12-01T23:29:01,317 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/9f6d020a52a74031b025d8a8f7436be3, entries=1, sequenceid=4, filesize=5.0 K 2024-12-01T23:29:01,318 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 7f5f4dca9fa6f9bee3b33a43f8499778 in 40ms, sequenceid=4, compaction requested=false 2024-12-01T23:29:01,319 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7f5f4dca9fa6f9bee3b33a43f8499778: 2024-12-01T23:29:01,320 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/82a9fd9563084f46a69b52fc917730e7_SeqId_4_ 2024-12-01T23:29:01,321 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_ 2024-12-01T23:29:01,322 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/4d29c59c06904efcba5f3846b421e25e_SeqId_4_ 2024-12-01T23:29:01,323 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile0 into 7f5f4dca9fa6f9bee3b33a43f8499778/a as hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/82a9fd9563084f46a69b52fc917730e7_SeqId_4_ - updating store file list. 2024-12-01T23:29:01,329 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 82a9fd9563084f46a69b52fc917730e7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T23:29:01,329 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/82a9fd9563084f46a69b52fc917730e7_SeqId_4_ into 7f5f4dca9fa6f9bee3b33a43f8499778/a 2024-12-01T23:29:01,329 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile0 into 7f5f4dca9fa6f9bee3b33a43f8499778/a (new location: hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/82a9fd9563084f46a69b52fc917730e7_SeqId_4_) 2024-12-01T23:29:01,331 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile1 into 7f5f4dca9fa6f9bee3b33a43f8499778/a as hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_ - updating store file list. 2024-12-01T23:29:01,336 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T23:29:01,337 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_ into 7f5f4dca9fa6f9bee3b33a43f8499778/a 2024-12-01T23:29:01,337 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile1 into 7f5f4dca9fa6f9bee3b33a43f8499778/a (new location: hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_) 2024-12-01T23:29:01,341 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile2 into 7f5f4dca9fa6f9bee3b33a43f8499778/a as hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/4d29c59c06904efcba5f3846b421e25e_SeqId_4_ - updating store file list. 2024-12-01T23:29:01,346 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4d29c59c06904efcba5f3846b421e25e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T23:29:01,346 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/4d29c59c06904efcba5f3846b421e25e_SeqId_4_ into 7f5f4dca9fa6f9bee3b33a43f8499778/a 2024-12-01T23:29:01,346 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35981/hbase/testCompactedBulkLoadedFiles/hfile2 into 7f5f4dca9fa6f9bee3b33a43f8499778/a (new location: hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/4d29c59c06904efcba5f3846b421e25e_SeqId_4_) 2024-12-01T23:29:01,357 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-01T23:29:01,357 DEBUG [Time-limited test {}] regionserver.HStore(1541): 7f5f4dca9fa6f9bee3b33a43f8499778/a is initiating major compaction (all files) 2024-12-01T23:29:01,357 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 7f5f4dca9fa6f9bee3b33a43f8499778/a in testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. 2024-12-01T23:29:01,358 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/9f6d020a52a74031b025d8a8f7436be3, hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/82a9fd9563084f46a69b52fc917730e7_SeqId_4_, hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_, hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/4d29c59c06904efcba5f3846b421e25e_SeqId_4_] into tmpdir=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/.tmp, totalSize=19.3 K 2024-12-01T23:29:01,359 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9f6d020a52a74031b025d8a8f7436be3, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733095741234 2024-12-01T23:29:01,359 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 82a9fd9563084f46a69b52fc917730e7_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-01T23:29:01,360 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-01T23:29:01,360 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4d29c59c06904efcba5f3846b421e25e_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-01T23:29:01,380 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/.tmp/a/684b758d63ea48139e11e8e8c450fa74 is 55, key is testCompactedBulkLoadedFiles/a:a/1733095741234/Put/seqid=0 2024-12-01T23:29:01,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741927_1105 (size=6154) 2024-12-01T23:29:01,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741927_1105 (size=6154) 2024-12-01T23:29:01,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741927_1105 (size=6154) 2024-12-01T23:29:01,412 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/.tmp/a/684b758d63ea48139e11e8e8c450fa74 as hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/684b758d63ea48139e11e8e8c450fa74 2024-12-01T23:29:01,425 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 7f5f4dca9fa6f9bee3b33a43f8499778/a of 7f5f4dca9fa6f9bee3b33a43f8499778 into 684b758d63ea48139e11e8e8c450fa74(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T23:29:01,425 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 7f5f4dca9fa6f9bee3b33a43f8499778: 2024-12-01T23:29:01,425 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-01T23:29:01,425 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-01T23:29:01,473 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178, size=0 (0bytes) 2024-12-01T23:29:01,473 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 might be still open, length is 0 2024-12-01T23:29:01,473 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 2024-12-01T23:29:01,473 WARN [IPC Server handler 1 on default port 35981 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-01T23:29:01,474 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 after 1ms 2024-12-01T23:29:02,044 INFO [master/f962de17fdc3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-01T23:29:02,044 INFO [master/f962de17fdc3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-01T23:29:04,232 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:39476 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:44369:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39476 dst: /127.0.0.1:44369 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44369 remote=/127.0.0.1:39476]. Total timeout mills is 60000, 57191 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:29:04,232 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:58530 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41911:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58530 dst: /127.0.0.1:41911 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:29:04,232 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:35834 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:39219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35834 dst: /127.0.0.1:39219 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:29:04,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741922_1106 (size=1794) 2024-12-01T23:29:04,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741922_1106 (size=1794) 2024-12-01T23:29:04,710 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T23:29:04,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-01T23:29:04,893 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-01T23:29:04,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-01T23:29:04,894 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-01T23:29:05,475 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 after 4002ms 2024-12-01T23:29:05,521 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:29:05,521 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 took 4049ms 2024-12-01T23:29:05,529 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178; continuing. 2024-12-01T23:29:05,529 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 so closing down 2024-12-01T23:29:05,529 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:29:05,552 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733095741178.temp 2024-12-01T23:29:05,598 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000003-wal.1733095741178.temp 2024-12-01T23:29:05,598 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:29:05,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741928_1107 (size=800) 2024-12-01T23:29:05,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741928_1107 (size=800) 2024-12-01T23:29:05,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741928_1107 (size=800) 2024-12-01T23:29:06,029 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000003-wal.1733095741178.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-01T23:29:06,030 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000003-wal.1733095741178.temp to hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000008 2024-12-01T23:29:06,031 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 507 ms; skipped=3; WAL=hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T23:29:06,031 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178, journal: Splitting hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178, size=0 (0bytes) at 1733095741473Finishing writing output for hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 so closing down at 1733095745529 (+4056 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000003-wal.1733095741178.temp at 1733095745598 (+69 ms)3 split writer threads finished at 1733095745598Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000003-wal.1733095741178.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733095746029 (+431 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000003-wal.1733095741178.temp to hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000008 at 1733095746031 (+2 ms)Processed 5 edits across 1 Regions in 507 ms; skipped=3; WAL=hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178, size=0, length=0, corrupted=false, cancelled=false at 1733095746031 2024-12-01T23:29:06,033 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 to hdfs://localhost:35981/hbase/oldWALs/wal.1733095741178 2024-12-01T23:29:06,049 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000008 2024-12-01T23:29:06,049 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:29:06,052 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:29:06,072 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095746053, exclude list is [], retry=0 2024-12-01T23:29:06,081 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:29:06,081 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:29:06,081 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:29:06,129 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095746053 2024-12-01T23:29:06,130 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:29:06,130 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f5f4dca9fa6f9bee3b33a43f8499778, NAME => 'testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:29:06,131 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:29:06,131 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,131 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,148 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,150 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5f4dca9fa6f9bee3b33a43f8499778 columnFamilyName a 2024-12-01T23:29:06,150 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:06,177 DEBUG [StoreFileOpener-7f5f4dca9fa6f9bee3b33a43f8499778-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4d29c59c06904efcba5f3846b421e25e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T23:29:06,177 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/4d29c59c06904efcba5f3846b421e25e_SeqId_4_ 2024-12-01T23:29:06,186 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/684b758d63ea48139e11e8e8c450fa74 2024-12-01T23:29:06,195 DEBUG [StoreFileOpener-7f5f4dca9fa6f9bee3b33a43f8499778-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 82a9fd9563084f46a69b52fc917730e7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T23:29:06,195 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/82a9fd9563084f46a69b52fc917730e7_SeqId_4_ 2024-12-01T23:29:06,204 DEBUG [StoreFileOpener-7f5f4dca9fa6f9bee3b33a43f8499778-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T23:29:06,204 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_ 2024-12-01T23:29:06,209 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/9f6d020a52a74031b025d8a8f7436be3 2024-12-01T23:29:06,209 WARN [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/4d29c59c06904efcba5f3846b421e25e_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6bad64c0 2024-12-01T23:29:06,209 WARN [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/82a9fd9563084f46a69b52fc917730e7_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6bad64c0 2024-12-01T23:29:06,209 WARN [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6bad64c0 2024-12-01T23:29:06,209 WARN [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/9f6d020a52a74031b025d8a8f7436be3 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6bad64c0 2024-12-01T23:29:06,209 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/4d29c59c06904efcba5f3846b421e25e_SeqId_4_, hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/82a9fd9563084f46a69b52fc917730e7_SeqId_4_, hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_, hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/9f6d020a52a74031b025d8a8f7436be3] to archive 2024-12-01T23:29:06,211 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T23:29:06,212 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/4d29c59c06904efcba5f3846b421e25e_SeqId_4_ to hdfs://localhost:35981/hbase/archive/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/4d29c59c06904efcba5f3846b421e25e_SeqId_4_ 2024-12-01T23:29:06,214 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/82a9fd9563084f46a69b52fc917730e7_SeqId_4_ to hdfs://localhost:35981/hbase/archive/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/82a9fd9563084f46a69b52fc917730e7_SeqId_4_ 2024-12-01T23:29:06,215 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_ to hdfs://localhost:35981/hbase/archive/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_ 2024-12-01T23:29:06,217 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/9f6d020a52a74031b025d8a8f7436be3 to hdfs://localhost:35981/hbase/archive/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/a/9f6d020a52a74031b025d8a8f7436be3 2024-12-01T23:29:06,218 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(327): Store=7f5f4dca9fa6f9bee3b33a43f8499778/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:06,218 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,219 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5f4dca9fa6f9bee3b33a43f8499778 columnFamilyName b 2024-12-01T23:29:06,219 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:06,219 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(327): Store=7f5f4dca9fa6f9bee3b33a43f8499778/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:06,219 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,220 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f5f4dca9fa6f9bee3b33a43f8499778 columnFamilyName c 2024-12-01T23:29:06,220 DEBUG [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:06,221 INFO [StoreOpener-7f5f4dca9fa6f9bee3b33a43f8499778-1 {}] regionserver.HStore(327): Store=7f5f4dca9fa6f9bee3b33a43f8499778/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:06,221 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,222 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,224 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000008 2024-12-01T23:29:06,255 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:29:06,258 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 7f5f4dca9fa6f9bee3b33a43f8499778 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "7f5f4dca9fa6f9bee3b33a43f8499778" family_name: "a" compaction_input: "9f6d020a52a74031b025d8a8f7436be3" compaction_input: "82a9fd9563084f46a69b52fc917730e7_SeqId_4_" compaction_input: "83b66f0b4aa34962a898d0addb7eacdb_SeqId_4_" compaction_input: "4d29c59c06904efcba5f3846b421e25e_SeqId_4_" compaction_output: "684b758d63ea48139e11e8e8c450fa74" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-01T23:29:06,258 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-01T23:29:06,259 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000008 2024-12-01T23:29:06,260 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/0000000000000000008 2024-12-01T23:29:06,262 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,262 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,263 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:29:06,264 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7f5f4dca9fa6f9bee3b33a43f8499778 2024-12-01T23:29:06,267 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testCompactedBulkLoadedFiles/7f5f4dca9fa6f9bee3b33a43f8499778/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-01T23:29:06,268 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7f5f4dca9fa6f9bee3b33a43f8499778; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71167843, jitterRate=0.060483500361442566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:29:06,268 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7f5f4dca9fa6f9bee3b33a43f8499778: Writing region info on filesystem at 1733095746131Initializing all the Stores at 1733095746140 (+9 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095746140Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095746141 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095746141Cleaning up temporary data from old regions at 1733095746262 (+121 ms)Region opened successfully at 1733095746268 (+6 ms) 2024-12-01T23:29:06,281 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7f5f4dca9fa6f9bee3b33a43f8499778, disabling compactions & flushes 2024-12-01T23:29:06,281 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. 2024-12-01T23:29:06,281 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. 2024-12-01T23:29:06,281 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. after waiting 0 ms 2024-12-01T23:29:06,281 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. 2024-12-01T23:29:06,285 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733095741101.7f5f4dca9fa6f9bee3b33a43f8499778. 2024-12-01T23:29:06,285 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7f5f4dca9fa6f9bee3b33a43f8499778: Waiting for close lock at 1733095746281Disabling compacts and flushes for region at 1733095746281Disabling writes for close at 1733095746281Writing region close event to WAL at 1733095746284 (+3 ms)Closed at 1733095746284 2024-12-01T23:29:06,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741929_1108 (size=175) 2024-12-01T23:29:06,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741929_1108 (size=175) 2024-12-01T23:29:06,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741929_1108 (size=175) 2024-12-01T23:29:06,313 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:29:06,313 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733095746053) 2024-12-01T23:29:06,334 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=435 (was 424) Potentially hanging thread: IPC Client (453912140) connection to localhost/127.0.0.1:46003 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1897381796_22 at /127.0.0.1:44780 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1897381796_22 at /127.0.0.1:42348 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1897381796_22 at /127.0.0.1:38126 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (453912140) connection to localhost/127.0.0.1:35981 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:35981 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46003 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1342 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1011 (was 1064), ProcessCount=11 (was 11), AvailableMemoryMB=3469 (was 3460) - AvailableMemoryMB LEAK? - 2024-12-01T23:29:06,335 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-12-01T23:29:06,354 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=435, OpenFileDescriptor=1342, MaxFileDescriptor=1048576, SystemLoadAverage=1011, ProcessCount=11, AvailableMemoryMB=3465 2024-12-01T23:29:06,354 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-12-01T23:29:06,368 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:29:06,370 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T23:29:06,371 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T23:29:06,373 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11432754, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/hregion-11432754, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:29:06,385 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11432754/hregion-11432754.1733095746373, exclude list is [], retry=0 2024-12-01T23:29:06,388 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:29:06,388 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:29:06,388 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:29:06,440 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11432754/hregion-11432754.1733095746373 2024-12-01T23:29:06,444 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:29:06,445 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8955c1f66258a8ebccff175336721dbb, NAME => 'testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35981/hbase 2024-12-01T23:29:06,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741931_1110 (size=67) 2024-12-01T23:29:06,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741931_1110 (size=67) 2024-12-01T23:29:06,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741931_1110 (size=67) 2024-12-01T23:29:06,460 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:29:06,464 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,466 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName a 2024-12-01T23:29:06,466 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:06,467 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:06,467 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,469 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName b 2024-12-01T23:29:06,469 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:06,470 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:06,470 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,472 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName c 2024-12-01T23:29:06,472 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:06,473 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:06,474 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,474 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,475 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,476 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,476 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,477 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:29:06,486 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,490 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T23:29:06,491 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8955c1f66258a8ebccff175336721dbb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73337041, jitterRate=0.09280706942081451}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:29:06,491 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8955c1f66258a8ebccff175336721dbb: Writing region info on filesystem at 1733095746460Initializing all the Stores at 1733095746461 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095746461Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095746464 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095746464Cleaning up temporary data from old regions at 1733095746476 (+12 ms)Region opened successfully at 1733095746491 (+15 ms) 2024-12-01T23:29:06,492 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8955c1f66258a8ebccff175336721dbb, disabling compactions & flushes 2024-12-01T23:29:06,492 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:06,492 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:06,492 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. after waiting 0 ms 2024-12-01T23:29:06,492 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:06,494 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:06,494 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8955c1f66258a8ebccff175336721dbb: Waiting for close lock at 1733095746492Disabling compacts and flushes for region at 1733095746492Disabling writes for close at 1733095746492Writing region close event to WAL at 1733095746494 (+2 ms)Closed at 1733095746494 2024-12-01T23:29:06,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741930_1109 (size=175) 2024-12-01T23:29:06,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741930_1109 (size=175) 2024-12-01T23:29:06,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741930_1109 (size=175) 2024-12-01T23:29:06,510 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:29:06,510 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11432754:(num 1733095746373) 2024-12-01T23:29:06,510 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:29:06,512 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:29:06,524 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512, exclude list is [], retry=0 2024-12-01T23:29:06,526 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:29:06,527 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:29:06,527 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:29:06,555 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512 2024-12-01T23:29:06,555 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:44317:44317)] 2024-12-01T23:29:06,556 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8955c1f66258a8ebccff175336721dbb, NAME => 'testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:29:06,556 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:29:06,556 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,556 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,557 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,558 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName a 2024-12-01T23:29:06,558 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:06,559 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:06,559 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,559 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName b 2024-12-01T23:29:06,559 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:06,560 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:06,560 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,561 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName c 2024-12-01T23:29:06,561 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:06,561 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:06,562 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,562 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,563 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,564 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,564 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,564 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:29:06,566 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:06,566 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8955c1f66258a8ebccff175336721dbb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72086659, jitterRate=0.0741749256849289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:29:06,566 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8955c1f66258a8ebccff175336721dbb: Writing region info on filesystem at 1733095746556Initializing all the Stores at 1733095746557 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095746557Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095746557Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095746557Cleaning up temporary data from old regions at 1733095746564 (+7 ms)Region opened successfully at 1733095746566 (+2 ms) 2024-12-01T23:29:06,576 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8955c1f66258a8ebccff175336721dbb 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-01T23:29:06,598 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/a/952787e6d0804069ad1d9193a80074bc is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733095746566/Put/seqid=0 2024-12-01T23:29:06,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741933_1112 (size=5958) 2024-12-01T23:29:06,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741933_1112 (size=5958) 2024-12-01T23:29:06,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741933_1112 (size=5958) 2024-12-01T23:29:07,020 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/a/952787e6d0804069ad1d9193a80074bc 2024-12-01T23:29:07,025 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/a/952787e6d0804069ad1d9193a80074bc as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/a/952787e6d0804069ad1d9193a80074bc 2024-12-01T23:29:07,029 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/a/952787e6d0804069ad1d9193a80074bc, entries=10, sequenceid=13, filesize=5.8 K 2024-12-01T23:29:07,030 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 8955c1f66258a8ebccff175336721dbb in 454ms, sequenceid=13, compaction requested=false 2024-12-01T23:29:07,030 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8955c1f66258a8ebccff175336721dbb: 2024-12-01T23:29:07,044 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8955c1f66258a8ebccff175336721dbb, disabling compactions & flushes 2024-12-01T23:29:07,044 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:07,044 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:07,044 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. after waiting 0 ms 2024-12-01T23:29:07,044 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:07,045 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:07,045 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:07,045 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8955c1f66258a8ebccff175336721dbb: Waiting for close lock at 1733095747044Disabling compacts and flushes for region at 1733095747044Disabling writes for close at 1733095747044Writing region close event to WAL at 1733095747045 (+1 ms)Closed at 1733095747045 2024-12-01T23:29:07,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741932_1111 (size=6243) 2024-12-01T23:29:07,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741932_1111 (size=6243) 2024-12-01T23:29:07,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741932_1111 (size=6243) 2024-12-01T23:29:07,062 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512, size=6.1 K (6243bytes) 2024-12-01T23:29:07,062 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512 2024-12-01T23:29:07,062 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512 after 0ms 2024-12-01T23:29:07,085 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:29:07,085 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512 took 23ms 2024-12-01T23:29:07,087 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512 so closing down 2024-12-01T23:29:07,088 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:29:07,089 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733095746512.temp 2024-12-01T23:29:07,111 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000003-wal.1733095746512.temp 2024-12-01T23:29:07,111 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:29:07,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741934_1113 (size=5575) 2024-12-01T23:29:07,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741934_1113 (size=5575) 2024-12-01T23:29:07,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741934_1113 (size=5575) 2024-12-01T23:29:07,117 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000003-wal.1733095746512.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-01T23:29:07,118 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000003-wal.1733095746512.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000035 2024-12-01T23:29:07,118 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 32 ms; skipped=2; WAL=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-01T23:29:07,119 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512, journal: Splitting hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512, size=6.1 K (6243bytes) at 1733095747062Finishing writing output for hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512 so closing down at 1733095747087 (+25 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000003-wal.1733095746512.temp at 1733095747111 (+24 ms)3 split writer threads finished at 1733095747111Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000003-wal.1733095746512.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733095747117 (+6 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000003-wal.1733095746512.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000035 at 1733095747118 (+1 ms)Processed 32 edits across 1 Regions in 32 ms; skipped=2; WAL=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733095747118 2024-12-01T23:29:07,120 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095746512 to hdfs://localhost:35981/hbase/oldWALs/wal.1733095746512 2024-12-01T23:29:07,120 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000035 2024-12-01T23:29:07,121 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:29:07,122 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:29:07,133 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122, exclude list is [], retry=0 2024-12-01T23:29:07,135 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:29:07,136 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:29:07,136 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:29:07,160 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 2024-12-01T23:29:07,160 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36067:36067),(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389)] 2024-12-01T23:29:07,160 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8955c1f66258a8ebccff175336721dbb, NAME => 'testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb.', STARTKEY => '', ENDKEY => ''} 2024-12-01T23:29:07,160 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:29:07,160 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,160 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,162 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,163 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName a 2024-12-01T23:29:07,163 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:07,168 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/a/952787e6d0804069ad1d9193a80074bc 2024-12-01T23:29:07,169 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:07,169 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,170 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName b 2024-12-01T23:29:07,171 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:07,171 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:07,171 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,172 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName c 2024-12-01T23:29:07,172 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:07,173 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:07,173 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,173 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,175 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,175 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000035 2024-12-01T23:29:07,199 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:29:07,200 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000035 2024-12-01T23:29:07,201 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8955c1f66258a8ebccff175336721dbb 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-01T23:29:07,214 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/b/a5f62dacb8ed4dd6b1d261e1b6b6431a is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733095747030/Put/seqid=0 2024-12-01T23:29:07,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741936_1115 (size=5958) 2024-12-01T23:29:07,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741936_1115 (size=5958) 2024-12-01T23:29:07,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741936_1115 (size=5958) 2024-12-01T23:29:07,220 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/b/a5f62dacb8ed4dd6b1d261e1b6b6431a 2024-12-01T23:29:07,237 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/c/de09c78ef7f04731a3c599742a3ec856 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733095747036/Put/seqid=0 2024-12-01T23:29:07,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741937_1116 (size=5958) 2024-12-01T23:29:07,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741937_1116 (size=5958) 2024-12-01T23:29:07,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741937_1116 (size=5958) 2024-12-01T23:29:07,244 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/c/de09c78ef7f04731a3c599742a3ec856 2024-12-01T23:29:07,249 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/b/a5f62dacb8ed4dd6b1d261e1b6b6431a as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/b/a5f62dacb8ed4dd6b1d261e1b6b6431a 2024-12-01T23:29:07,254 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/b/a5f62dacb8ed4dd6b1d261e1b6b6431a, entries=10, sequenceid=35, filesize=5.8 K 2024-12-01T23:29:07,255 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/c/de09c78ef7f04731a3c599742a3ec856 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/c/de09c78ef7f04731a3c599742a3ec856 2024-12-01T23:29:07,260 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/c/de09c78ef7f04731a3c599742a3ec856, entries=10, sequenceid=35, filesize=5.8 K 2024-12-01T23:29:07,260 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 8955c1f66258a8ebccff175336721dbb in 59ms, sequenceid=35, compaction requested=false; wal=null 2024-12-01T23:29:07,261 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000035 2024-12-01T23:29:07,262 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,262 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,263 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:29:07,264 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:07,266 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-01T23:29:07,267 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8955c1f66258a8ebccff175336721dbb; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73656574, jitterRate=0.09756848216056824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:29:07,267 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8955c1f66258a8ebccff175336721dbb: Writing region info on filesystem at 1733095747160Initializing all the Stores at 1733095747161 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095747162 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095747162Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095747162Obtaining lock to block concurrent updates at 1733095747201 (+39 ms)Preparing flush snapshotting stores in 8955c1f66258a8ebccff175336721dbb at 1733095747201Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733095747201Flushing stores of testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. at 1733095747201Flushing 8955c1f66258a8ebccff175336721dbb/b: creating writer at 1733095747201Flushing 8955c1f66258a8ebccff175336721dbb/b: appending metadata at 1733095747213 (+12 ms)Flushing 8955c1f66258a8ebccff175336721dbb/b: closing flushed file at 1733095747213Flushing 8955c1f66258a8ebccff175336721dbb/c: creating writer at 1733095747225 (+12 ms)Flushing 8955c1f66258a8ebccff175336721dbb/c: appending metadata at 1733095747237 (+12 ms)Flushing 8955c1f66258a8ebccff175336721dbb/c: closing flushed file at 1733095747237Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12550c23: reopening flushed file at 1733095747248 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a5ff26a: reopening flushed file at 1733095747254 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 8955c1f66258a8ebccff175336721dbb in 59ms, sequenceid=35, compaction requested=false; wal=null at 1733095747260 (+6 ms)Cleaning up temporary data from old regions at 1733095747262 (+2 ms)Region opened successfully at 1733095747267 (+5 ms) 2024-12-01T23:29:07,333 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122, size=0 (0bytes) 2024-12-01T23:29:07,333 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 might be still open, length is 0 2024-12-01T23:29:07,333 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 2024-12-01T23:29:07,334 WARN [IPC Server handler 4 on default port 35981 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-01T23:29:07,334 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 after 1ms 2024-12-01T23:29:10,233 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:44834 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:39219:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44834 dst: /127.0.0.1:39219 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39219 remote=/127.0.0.1:44834]. Total timeout mills is 60000, 57067 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:29:10,234 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:38180 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41911:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38180 dst: /127.0.0.1:41911 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:29:10,234 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1840701859_22 at /127.0.0.1:42414 [Receiving block BP-959958609-172.17.0.3-1733095688324:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:44369:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42414 dst: /127.0.0.1:44369 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:29:10,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741935_1117 (size=5567) 2024-12-01T23:29:10,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741935_1117 (size=5567) 2024-12-01T23:29:10,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741935_1117 (size=5567) 2024-12-01T23:29:11,335 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 after 4002ms 2024-12-01T23:29:11,374 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:29:11,374 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 took 4041ms 2024-12-01T23:29:11,387 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122; continuing. 2024-12-01T23:29:11,387 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 so closing down 2024-12-01T23:29:11,387 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T23:29:11,397 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733095747122.temp 2024-12-01T23:29:11,450 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000037-wal.1733095747122.temp 2024-12-01T23:29:11,450 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T23:29:11,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741938_1118 (size=5575) 2024-12-01T23:29:11,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741938_1118 (size=5575) 2024-12-01T23:29:11,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741938_1118 (size=5575) 2024-12-01T23:29:11,492 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000037-wal.1733095747122.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-01T23:29:11,496 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000037-wal.1733095747122.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000066 2024-12-01T23:29:11,497 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 121 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T23:29:11,498 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122, journal: Splitting hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122, size=0 (0bytes) at 1733095747333Finishing writing output for hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 so closing down at 1733095751387 (+4054 ms)Creating recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000037-wal.1733095747122.temp at 1733095751450 (+63 ms)3 split writer threads finished at 1733095751450Closed recovered edits writer path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000037-wal.1733095747122.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733095751492 (+42 ms)Rename recovered edits hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000037-wal.1733095747122.temp to hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000066 at 1733095751496 (+4 ms)Processed 30 edits across 1 Regions in 121 ms; skipped=0; WAL=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122, size=0, length=0, corrupted=false, cancelled=false at 1733095751497 (+1 ms) 2024-12-01T23:29:11,501 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 to hdfs://localhost:35981/hbase/oldWALs/wal.1733095747122 2024-12-01T23:29:11,502 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000066 2024-12-01T23:29:11,502 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T23:29:11,505 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35981/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367, archiveDir=hdfs://localhost:35981/hbase/oldWALs, maxLogs=32 2024-12-01T23:29:11,524 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095751505, exclude list is [], retry=0 2024-12-01T23:29:11,529 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44369,DS-23f7379b-d6a8-4394-9494-ce51b3d3d379,DISK] 2024-12-01T23:29:11,529 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41911,DS-0b73283e-be70-47e5-a2bb-bfd83dc14319,DISK] 2024-12-01T23:29:11,530 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39219,DS-2d3c3996-609e-4c25-8125-ae89bae8cd3c,DISK] 2024-12-01T23:29:11,592 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095751505 2024-12-01T23:29:11,597 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44317:44317),(127.0.0.1/127.0.0.1:37389:37389),(127.0.0.1/127.0.0.1:36067:36067)] 2024-12-01T23:29:11,598 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T23:29:11,612 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:11,619 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName a 2024-12-01T23:29:11,619 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:11,651 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/a/952787e6d0804069ad1d9193a80074bc 2024-12-01T23:29:11,651 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:11,651 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:11,660 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName b 2024-12-01T23:29:11,661 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:11,704 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/b/a5f62dacb8ed4dd6b1d261e1b6b6431a 2024-12-01T23:29:11,704 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:11,704 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:11,709 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8955c1f66258a8ebccff175336721dbb columnFamilyName c 2024-12-01T23:29:11,709 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T23:29:11,778 DEBUG [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/c/de09c78ef7f04731a3c599742a3ec856 2024-12-01T23:29:11,779 INFO [StoreOpener-8955c1f66258a8ebccff175336721dbb-1 {}] regionserver.HStore(327): Store=8955c1f66258a8ebccff175336721dbb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T23:29:11,780 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:11,790 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:11,813 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:11,828 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000066 2024-12-01T23:29:11,890 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-01T23:29:11,900 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000066 2024-12-01T23:29:11,901 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8955c1f66258a8ebccff175336721dbb 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-01T23:29:11,938 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/a/30bbd89c1fdc493b93f3af6c4a353701 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733095747275/Put/seqid=0 2024-12-01T23:29:12,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741940_1120 (size=5958) 2024-12-01T23:29:12,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741940_1120 (size=5958) 2024-12-01T23:29:12,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741940_1120 (size=5958) 2024-12-01T23:29:12,014 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/a/30bbd89c1fdc493b93f3af6c4a353701 2024-12-01T23:29:12,055 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/b/7c0669984d6f498f9be2421c3e17f577 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733095747287/Put/seqid=0 2024-12-01T23:29:12,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741941_1121 (size=5958) 2024-12-01T23:29:12,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741941_1121 (size=5958) 2024-12-01T23:29:12,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741941_1121 (size=5958) 2024-12-01T23:29:12,559 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/b/7c0669984d6f498f9be2421c3e17f577 2024-12-01T23:29:12,600 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/c/5e1216e83f3545b3b0c4f1ddb504513d is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733095747293/Put/seqid=0 2024-12-01T23:29:12,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741942_1122 (size=5958) 2024-12-01T23:29:12,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741942_1122 (size=5958) 2024-12-01T23:29:12,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741942_1122 (size=5958) 2024-12-01T23:29:13,049 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/c/5e1216e83f3545b3b0c4f1ddb504513d 2024-12-01T23:29:13,056 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/a/30bbd89c1fdc493b93f3af6c4a353701 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/a/30bbd89c1fdc493b93f3af6c4a353701 2024-12-01T23:29:13,065 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/a/30bbd89c1fdc493b93f3af6c4a353701, entries=10, sequenceid=66, filesize=5.8 K 2024-12-01T23:29:13,067 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/b/7c0669984d6f498f9be2421c3e17f577 as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/b/7c0669984d6f498f9be2421c3e17f577 2024-12-01T23:29:13,073 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/b/7c0669984d6f498f9be2421c3e17f577, entries=10, sequenceid=66, filesize=5.8 K 2024-12-01T23:29:13,075 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/.tmp/c/5e1216e83f3545b3b0c4f1ddb504513d as hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/c/5e1216e83f3545b3b0c4f1ddb504513d 2024-12-01T23:29:13,082 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/c/5e1216e83f3545b3b0c4f1ddb504513d, entries=10, sequenceid=66, filesize=5.8 K 2024-12-01T23:29:13,083 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 8955c1f66258a8ebccff175336721dbb in 1182ms, sequenceid=66, compaction requested=false; wal=null 2024-12-01T23:29:13,084 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/0000000000000000066 2024-12-01T23:29:13,086 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:13,086 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:13,087 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T23:29:13,089 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8955c1f66258a8ebccff175336721dbb 2024-12-01T23:29:13,102 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/hbase/data/default/testReplayEditsWrittenViaHRegion/8955c1f66258a8ebccff175336721dbb/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-01T23:29:13,106 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8955c1f66258a8ebccff175336721dbb; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61599183, jitterRate=-0.08210064470767975}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T23:29:13,107 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8955c1f66258a8ebccff175336721dbb: Writing region info on filesystem at 1733095751598Initializing all the Stores at 1733095751608 (+10 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095751608Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095751611 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733095751611Obtaining lock to block concurrent updates at 1733095751901 (+290 ms)Preparing flush snapshotting stores in 8955c1f66258a8ebccff175336721dbb at 1733095751901Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733095751901Flushing stores of testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. at 1733095751901Flushing 8955c1f66258a8ebccff175336721dbb/a: creating writer at 1733095751901Flushing 8955c1f66258a8ebccff175336721dbb/a: appending metadata at 1733095751936 (+35 ms)Flushing 8955c1f66258a8ebccff175336721dbb/a: closing flushed file at 1733095751937 (+1 ms)Flushing 8955c1f66258a8ebccff175336721dbb/b: creating writer at 1733095752027 (+90 ms)Flushing 8955c1f66258a8ebccff175336721dbb/b: appending metadata at 1733095752054 (+27 ms)Flushing 8955c1f66258a8ebccff175336721dbb/b: closing flushed file at 1733095752054Flushing 8955c1f66258a8ebccff175336721dbb/c: creating writer at 1733095752565 (+511 ms)Flushing 8955c1f66258a8ebccff175336721dbb/c: appending metadata at 1733095752599 (+34 ms)Flushing 8955c1f66258a8ebccff175336721dbb/c: closing flushed file at 1733095752599Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1878046b: reopening flushed file at 1733095753055 (+456 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64414d87: reopening flushed file at 1733095753065 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52e97aa1: reopening flushed file at 1733095753074 (+9 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 8955c1f66258a8ebccff175336721dbb in 1182ms, sequenceid=66, compaction requested=false; wal=null at 1733095753083 (+9 ms)Cleaning up temporary data from old regions at 1733095753086 (+3 ms)Region opened successfully at 1733095753106 (+20 ms) 2024-12-01T23:29:13,138 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8955c1f66258a8ebccff175336721dbb, disabling compactions & flushes 2024-12-01T23:29:13,138 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:13,138 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:13,138 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. after waiting 0 ms 2024-12-01T23:29:13,138 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:13,152 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733095746369.8955c1f66258a8ebccff175336721dbb. 2024-12-01T23:29:13,153 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8955c1f66258a8ebccff175336721dbb: Waiting for close lock at 1733095753138Disabling compacts and flushes for region at 1733095753138Disabling writes for close at 1733095753138Writing region close event to WAL at 1733095753152 (+14 ms)Closed at 1733095753152 2024-12-01T23:29:13,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741939_1119 (size=175) 2024-12-01T23:29:13,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741939_1119 (size=175) 2024-12-01T23:29:13,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741939_1119 (size=175) 2024-12-01T23:29:13,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T23:29:13,177 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733095751505) 2024-12-01T23:29:13,197 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=433 (was 435), OpenFileDescriptor=1404 (was 1342) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=978 (was 1011), ProcessCount=11 (was 11), AvailableMemoryMB=2753 (was 3465) 2024-12-01T23:29:13,197 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1404 is superior to 1024 2024-12-01T23:29:13,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-01T23:29:13,198 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T23:29:13,198 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:29:13,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:29:13,199 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:29:13,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T23:29:13,200 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-01T23:29:13,200 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1656547772, stopped=false 2024-12-01T23:29:13,201 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f962de17fdc3,36351,1733095694462 2024-12-01T23:29:13,309 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T23:29:13,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T23:29:13,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T23:29:13,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T23:29:13,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:29:13,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:29:13,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:29:13,435 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T23:29:13,435 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T23:29:13,435 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:29:13,436 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:29:13,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:29:13,436 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f962de17fdc3,34145,1733095695383' ***** 2024-12-01T23:29:13,436 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T23:29:13,436 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f962de17fdc3,33945,1733095695537' ***** 2024-12-01T23:29:13,436 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T23:29:13,436 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T23:29:13,436 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T23:29:13,436 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T23:29:13,436 INFO [RS:1;f962de17fdc3:33945 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T23:29:13,436 INFO [RS:0;f962de17fdc3:34145 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T23:29:13,436 INFO [RS:0;f962de17fdc3:34145 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T23:29:13,436 INFO [RS:1;f962de17fdc3:33945 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T23:29:13,437 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(3091): Received CLOSE for 153d48efbf4361bd73ec060241e37021 2024-12-01T23:29:13,437 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(959): stopping server f962de17fdc3,33945,1733095695537 2024-12-01T23:29:13,437 INFO [RS:1;f962de17fdc3:33945 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:29:13,437 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(959): stopping server f962de17fdc3,34145,1733095695383 2024-12-01T23:29:13,437 INFO [RS:0;f962de17fdc3:34145 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:29:13,437 INFO [RS:1;f962de17fdc3:33945 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f962de17fdc3:33945. 2024-12-01T23:29:13,437 INFO [RS:0;f962de17fdc3:34145 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f962de17fdc3:34145. 2024-12-01T23:29:13,437 DEBUG [RS:1;f962de17fdc3:33945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:29:13,437 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 153d48efbf4361bd73ec060241e37021, disabling compactions & flushes 2024-12-01T23:29:13,437 DEBUG [RS:0;f962de17fdc3:34145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T23:29:13,437 DEBUG [RS:1;f962de17fdc3:33945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:29:13,437 DEBUG [RS:0;f962de17fdc3:34145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:29:13,437 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:29:13,437 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:29:13,437 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T23:29:13,437 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(976): stopping server f962de17fdc3,33945,1733095695537; all regions closed. 2024-12-01T23:29:13,437 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. after waiting 0 ms 2024-12-01T23:29:13,437 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:29:13,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:29:13,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T23:29:13,438 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T23:29:13,438 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T23:29:13,438 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T23:29:13,438 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T23:29:13,439 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-01T23:29:13,439 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 153d48efbf4361bd73ec060241e37021=testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021.} 2024-12-01T23:29:13,439 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1351): Waiting on 153d48efbf4361bd73ec060241e37021, 1588230740 2024-12-01T23:29:13,439 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T23:29:13,439 INFO [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T23:29:13,439 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T23:29:13,439 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T23:29:13,439 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T23:29:13,439 INFO [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-01T23:29:13,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741833_1009 (size=175) 2024-12-01T23:29:13,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741833_1009 (size=175) 2024-12-01T23:29:13,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741833_1009 (size=175) 2024-12-01T23:29:13,445 INFO [regionserver/f962de17fdc3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:29:13,451 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/info/4839fb3bcd384616a8e1fb7c477a01b8 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021./info:regioninfo/1733095728409/Put/seqid=0 2024-12-01T23:29:13,451 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/default/testReplayEditsAfterRegionMovedWithMultiCF/153d48efbf4361bd73ec060241e37021/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-01T23:29:13,453 INFO [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:29:13,453 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 153d48efbf4361bd73ec060241e37021: Waiting for close lock at 1733095753437Running coprocessor pre-close hooks at 1733095753437Disabling compacts and flushes for region at 1733095753437Disabling writes for close at 1733095753437Writing region close event to WAL at 1733095753439 (+2 ms)Running coprocessor post-close hooks at 1733095753452 (+13 ms)Closed at 1733095753453 (+1 ms) 2024-12-01T23:29:13,453 DEBUG [RS_CLOSE_REGION-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733095721334.153d48efbf4361bd73ec060241e37021. 2024-12-01T23:29:13,453 DEBUG [RS:1;f962de17fdc3:33945 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/oldWALs 2024-12-01T23:29:13,453 INFO [RS:1;f962de17fdc3:33945 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f962de17fdc3%2C33945%2C1733095695537:(num 1733095700474) 2024-12-01T23:29:13,453 DEBUG [RS:1;f962de17fdc3:33945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:29:13,453 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:29:13,453 INFO [RS:1;f962de17fdc3:33945 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:29:13,453 INFO [RS:1;f962de17fdc3:33945 {}] hbase.ChoreService(370): Chore service for: regionserver/f962de17fdc3:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-01T23:29:13,454 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T23:29:13,454 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T23:29:13,454 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T23:29:13,454 INFO [RS:1;f962de17fdc3:33945 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:29:13,454 INFO [RS:1;f962de17fdc3:33945 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33945 2024-12-01T23:29:13,454 INFO [regionserver/f962de17fdc3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:29:13,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741943_1123 (size=6778) 2024-12-01T23:29:13,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741943_1123 (size=6778) 2024-12-01T23:29:13,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741943_1123 (size=6778) 2024-12-01T23:29:13,468 INFO [regionserver/f962de17fdc3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:29:13,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f962de17fdc3,33945,1733095695537 2024-12-01T23:29:13,638 INFO [RS:1;f962de17fdc3:33945 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:29:13,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:29:13,639 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-01T23:29:13,777 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f962de17fdc3,33945,1733095695537] 2024-12-01T23:29:13,839 DEBUG [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-01T23:29:13,868 INFO [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/info/4839fb3bcd384616a8e1fb7c477a01b8 2024-12-01T23:29:13,877 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/.tmp/info/4839fb3bcd384616a8e1fb7c477a01b8 as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/info/4839fb3bcd384616a8e1fb7c477a01b8 2024-12-01T23:29:13,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:29:13,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33945-0x101933997fc0002, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:29:13,878 INFO [RS:1;f962de17fdc3:33945 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:29:13,878 INFO [RS:1;f962de17fdc3:33945 {}] regionserver.HRegionServer(1031): Exiting; stopping=f962de17fdc3,33945,1733095695537; zookeeper connection closed. 2024-12-01T23:29:13,878 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@783a29d8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@783a29d8 2024-12-01T23:29:13,885 INFO [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/info/4839fb3bcd384616a8e1fb7c477a01b8, entries=8, sequenceid=23, filesize=6.6 K 2024-12-01T23:29:13,886 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f962de17fdc3,33945,1733095695537 already deleted, retry=false 2024-12-01T23:29:13,886 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f962de17fdc3,33945,1733095695537 expired; onlineServers=1 2024-12-01T23:29:13,886 INFO [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 447ms, sequenceid=23, compaction requested=false 2024-12-01T23:29:13,886 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-01T23:29:13,912 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-01T23:29:13,913 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T23:29:13,913 INFO [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T23:29:13,913 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733095753439Running coprocessor pre-close hooks at 1733095753439Disabling compacts and flushes for region at 1733095753439Disabling writes for close at 1733095753439Obtaining lock to block concurrent updates at 1733095753439Preparing flush snapshotting stores in 1588230740 at 1733095753439Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733095753439Flushing stores of hbase:meta,,1.1588230740 at 1733095753440 (+1 ms)Flushing 1588230740/info: creating writer at 1733095753440Flushing 1588230740/info: appending metadata at 1733095753450 (+10 ms)Flushing 1588230740/info: closing flushed file at 1733095753451 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74aca435: reopening flushed file at 1733095753876 (+425 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 447ms, sequenceid=23, compaction requested=false at 1733095753886 (+10 ms)Writing region close event to WAL at 1733095753905 (+19 ms)Running coprocessor post-close hooks at 1733095753912 (+7 ms)Closed at 1733095753913 (+1 ms) 2024-12-01T23:29:13,913 DEBUG [RS_CLOSE_META-regionserver/f962de17fdc3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T23:29:14,039 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(976): stopping server f962de17fdc3,34145,1733095695383; all regions closed. 2024-12-01T23:29:14,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741893_1071 (size=2625) 2024-12-01T23:29:14,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741893_1071 (size=2625) 2024-12-01T23:29:14,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741893_1071 (size=2625) 2024-12-01T23:29:14,061 DEBUG [RS:0;f962de17fdc3:34145 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/oldWALs 2024-12-01T23:29:14,061 INFO [RS:0;f962de17fdc3:34145 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f962de17fdc3%2C34145%2C1733095695383.meta:.meta(num 1733095727112) 2024-12-01T23:29:14,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741835_1011 (size=3621) 2024-12-01T23:29:14,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741835_1011 (size=3621) 2024-12-01T23:29:14,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741835_1011 (size=3621) 2024-12-01T23:29:14,102 DEBUG [RS:0;f962de17fdc3:34145 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/oldWALs 2024-12-01T23:29:14,102 INFO [RS:0;f962de17fdc3:34145 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f962de17fdc3%2C34145%2C1733095695383:(num 1733095700525) 2024-12-01T23:29:14,102 DEBUG [RS:0;f962de17fdc3:34145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T23:29:14,103 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T23:29:14,104 INFO [RS:0;f962de17fdc3:34145 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:29:14,104 INFO [RS:0;f962de17fdc3:34145 {}] hbase.ChoreService(370): Chore service for: regionserver/f962de17fdc3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T23:29:14,104 INFO [RS:0;f962de17fdc3:34145 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:29:14,104 INFO [regionserver/f962de17fdc3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:29:14,104 INFO [RS:0;f962de17fdc3:34145 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34145 2024-12-01T23:29:14,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f962de17fdc3,34145,1733095695383 2024-12-01T23:29:14,235 INFO [RS:0;f962de17fdc3:34145 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:29:14,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T23:29:14,251 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f962de17fdc3,34145,1733095695383] 2024-12-01T23:29:14,261 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f962de17fdc3,34145,1733095695383 already deleted, retry=false 2024-12-01T23:29:14,261 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f962de17fdc3,34145,1733095695383 expired; onlineServers=0 2024-12-01T23:29:14,261 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f962de17fdc3,36351,1733095694462' ***** 2024-12-01T23:29:14,261 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-01T23:29:14,262 INFO [M:0;f962de17fdc3:36351 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T23:29:14,262 INFO [M:0;f962de17fdc3:36351 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T23:29:14,262 DEBUG [M:0;f962de17fdc3:36351 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-01T23:29:14,262 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-01T23:29:14,262 DEBUG [M:0;f962de17fdc3:36351 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-01T23:29:14,262 DEBUG [master/f962de17fdc3:0:becomeActiveMaster-HFileCleaner.large.0-1733095699428 {}] cleaner.HFileCleaner(306): Exit Thread[master/f962de17fdc3:0:becomeActiveMaster-HFileCleaner.large.0-1733095699428,5,FailOnTimeoutGroup] 2024-12-01T23:29:14,262 DEBUG [master/f962de17fdc3:0:becomeActiveMaster-HFileCleaner.small.0-1733095699434 {}] cleaner.HFileCleaner(306): Exit Thread[master/f962de17fdc3:0:becomeActiveMaster-HFileCleaner.small.0-1733095699434,5,FailOnTimeoutGroup] 2024-12-01T23:29:14,262 INFO [M:0;f962de17fdc3:36351 {}] hbase.ChoreService(370): Chore service for: master/f962de17fdc3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-01T23:29:14,262 INFO [M:0;f962de17fdc3:36351 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T23:29:14,262 DEBUG [M:0;f962de17fdc3:36351 {}] master.HMaster(1795): Stopping service threads 2024-12-01T23:29:14,262 INFO [M:0;f962de17fdc3:36351 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-01T23:29:14,263 INFO [M:0;f962de17fdc3:36351 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T23:29:14,263 INFO [M:0;f962de17fdc3:36351 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-01T23:29:14,263 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-01T23:29:14,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-01T23:29:14,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T23:29:14,272 DEBUG [M:0;f962de17fdc3:36351 {}] zookeeper.ZKUtil(347): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-01T23:29:14,272 WARN [M:0;f962de17fdc3:36351 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-01T23:29:14,273 INFO [M:0;f962de17fdc3:36351 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/.lastflushedseqids 2024-12-01T23:29:14,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741944_1124 (size=119) 2024-12-01T23:29:14,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741944_1124 (size=119) 2024-12-01T23:29:14,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741944_1124 (size=119) 2024-12-01T23:29:14,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:29:14,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34145-0x101933997fc0001, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:29:14,351 INFO [RS:0;f962de17fdc3:34145 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:29:14,351 INFO [RS:0;f962de17fdc3:34145 {}] regionserver.HRegionServer(1031): Exiting; stopping=f962de17fdc3,34145,1733095695383; zookeeper connection closed. 2024-12-01T23:29:14,352 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@74175a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@74175a 2024-12-01T23:29:14,352 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-01T23:29:14,710 INFO [M:0;f962de17fdc3:36351 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-01T23:29:14,711 INFO [M:0;f962de17fdc3:36351 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-01T23:29:14,711 DEBUG [M:0;f962de17fdc3:36351 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T23:29:14,711 INFO [M:0;f962de17fdc3:36351 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:29:14,711 DEBUG [M:0;f962de17fdc3:36351 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:29:14,711 DEBUG [M:0;f962de17fdc3:36351 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T23:29:14,711 DEBUG [M:0;f962de17fdc3:36351 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:29:14,711 INFO [M:0;f962de17fdc3:36351 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-12-01T23:29:14,743 DEBUG [M:0;f962de17fdc3:36351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f0fcf572bae848068ec82813bb7567ae is 82, key is hbase:meta,,1/info:regioninfo/1733095727500/Put/seqid=0 2024-12-01T23:29:14,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741945_1125 (size=6063) 2024-12-01T23:29:14,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741945_1125 (size=6063) 2024-12-01T23:29:14,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741945_1125 (size=6063) 2024-12-01T23:29:14,772 INFO [M:0;f962de17fdc3:36351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f0fcf572bae848068ec82813bb7567ae 2024-12-01T23:29:14,813 DEBUG [M:0;f962de17fdc3:36351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fad765f226e0423f96c61cc67ab858b4 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733095722409/Put/seqid=0 2024-12-01T23:29:14,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741946_1126 (size=7907) 2024-12-01T23:29:14,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741946_1126 (size=7907) 2024-12-01T23:29:14,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741946_1126 (size=7907) 2024-12-01T23:29:14,852 INFO [M:0;f962de17fdc3:36351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fad765f226e0423f96c61cc67ab858b4 2024-12-01T23:29:14,862 INFO [M:0;f962de17fdc3:36351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fad765f226e0423f96c61cc67ab858b4 2024-12-01T23:29:14,890 DEBUG [M:0;f962de17fdc3:36351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c42d6a33a084368ad5618a696d41f79 is 69, key is f962de17fdc3,33945,1733095695537/rs:state/1733095699687/Put/seqid=0 2024-12-01T23:29:14,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T23:29:14,893 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-01T23:29:14,894 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-01T23:29:14,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741947_1127 (size=5440) 2024-12-01T23:29:14,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741947_1127 (size=5440) 2024-12-01T23:29:14,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741947_1127 (size=5440) 2024-12-01T23:29:14,930 INFO [M:0;f962de17fdc3:36351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c42d6a33a084368ad5618a696d41f79 2024-12-01T23:29:14,950 INFO [M:0;f962de17fdc3:36351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8c42d6a33a084368ad5618a696d41f79 2024-12-01T23:29:14,953 DEBUG [M:0;f962de17fdc3:36351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f0fcf572bae848068ec82813bb7567ae as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f0fcf572bae848068ec82813bb7567ae 2024-12-01T23:29:14,959 INFO [M:0;f962de17fdc3:36351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f0fcf572bae848068ec82813bb7567ae, entries=14, sequenceid=207, filesize=5.9 K 2024-12-01T23:29:14,960 DEBUG [M:0;f962de17fdc3:36351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fad765f226e0423f96c61cc67ab858b4 as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fad765f226e0423f96c61cc67ab858b4 2024-12-01T23:29:14,965 INFO [M:0;f962de17fdc3:36351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fad765f226e0423f96c61cc67ab858b4 2024-12-01T23:29:14,965 INFO [M:0;f962de17fdc3:36351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fad765f226e0423f96c61cc67ab858b4, entries=21, sequenceid=207, filesize=7.7 K 2024-12-01T23:29:14,966 DEBUG [M:0;f962de17fdc3:36351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c42d6a33a084368ad5618a696d41f79 as hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8c42d6a33a084368ad5618a696d41f79 2024-12-01T23:29:14,981 INFO [M:0;f962de17fdc3:36351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8c42d6a33a084368ad5618a696d41f79 2024-12-01T23:29:14,981 INFO [M:0;f962de17fdc3:36351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35981/user/jenkins/test-data/359e7393-a626-1eab-c278-8e648ad597a0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8c42d6a33a084368ad5618a696d41f79, entries=3, sequenceid=207, filesize=5.3 K 2024-12-01T23:29:14,983 INFO [M:0;f962de17fdc3:36351 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 272ms, sequenceid=207, compaction requested=false 2024-12-01T23:29:14,984 INFO [M:0;f962de17fdc3:36351 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T23:29:14,985 DEBUG [M:0;f962de17fdc3:36351 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733095754711Disabling compacts and flushes for region at 1733095754711Disabling writes for close at 1733095754711Obtaining lock to block concurrent updates at 1733095754711Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733095754711Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1733095754712 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733095754713 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733095754713Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733095754743 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733095754743Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733095754786 (+43 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733095754812 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733095754812Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733095754862 (+50 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733095754889 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733095754889Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a2f3669: reopening flushed file at 1733095754950 (+61 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4643bbff: reopening flushed file at 1733095754959 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a560cd7: reopening flushed file at 1733095754965 (+6 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 272ms, sequenceid=207, compaction requested=false at 1733095754983 (+18 ms)Writing region close event to WAL at 1733095754984 (+1 ms)Closed at 1733095754984 2024-12-01T23:29:14,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39219 is added to blk_1073741830_1006 (size=102162) 2024-12-01T23:29:14,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44369 is added to blk_1073741830_1006 (size=102162) 2024-12-01T23:29:14,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41911 is added to blk_1073741830_1006 (size=102162) 2024-12-01T23:29:14,992 INFO [M:0;f962de17fdc3:36351 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-01T23:29:14,992 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T23:29:14,992 INFO [M:0;f962de17fdc3:36351 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36351 2024-12-01T23:29:14,992 INFO [M:0;f962de17fdc3:36351 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T23:29:15,144 INFO [M:0;f962de17fdc3:36351 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T23:29:15,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:29:15,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36351-0x101933997fc0000, quorum=127.0.0.1:56655, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T23:29:15,153 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095739298 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095739298 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1840701859_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733095729876/wal.1733095739298 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1840701859_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-01T23:29:15,157 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729683 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729683 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1840701859_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733095728660/wal.1733095729683 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1840701859_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-01T23:29:15,158 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733095716159/wal.1733095716413 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:29:15,162 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095740437 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095740437 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1840701859_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733095739654/wal.1733095740437 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1840701859_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-01T23:29:15,163 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733095741100/wal.1733095741178 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:29:15,163 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733095746367/wal.1733095747122 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:29:15,164 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733095704825/wal.1733095704987 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T23:29:15,169 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal.1733095704592 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal.1733095704592 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1840701859_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733095704164/wal.1733095704592 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1840701859_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-01T23:29:15,177 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal.1733095703458 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal.1733095703458 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1840701859_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733095702268/wal.1733095703458 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1840701859_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-01T23:29:15,193 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2df1c2c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T23:29:15,197 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1cfa1d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T23:29:15,197 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T23:29:15,197 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75279722{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T23:29:15,197 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76be007d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/hadoop.log.dir/,STOPPED} 2024-12-01T23:29:15,222 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T23:29:15,222 WARN [BP-959958609-172.17.0.3-1733095688324 heartbeating to localhost/127.0.0.1:35981 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T23:29:15,222 WARN [BP-959958609-172.17.0.3-1733095688324 heartbeating to localhost/127.0.0.1:35981 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-959958609-172.17.0.3-1733095688324 (Datanode Uuid 600d60c5-46af-4ad8-97d9-3329e790967e) service to localhost/127.0.0.1:35981 2024-12-01T23:29:15,222 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T23:29:15,223 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data5/current/BP-959958609-172.17.0.3-1733095688324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T23:29:15,223 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data6/current/BP-959958609-172.17.0.3-1733095688324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T23:29:15,224 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T23:29:15,229 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@58fd25ac{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T23:29:15,230 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c7bf474{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T23:29:15,230 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T23:29:15,230 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2433eef4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T23:29:15,230 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34eaca1d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/hadoop.log.dir/,STOPPED} 2024-12-01T23:29:15,233 WARN [BP-959958609-172.17.0.3-1733095688324 heartbeating to localhost/127.0.0.1:35981 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T23:29:15,233 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T23:29:15,233 WARN [BP-959958609-172.17.0.3-1733095688324 heartbeating to localhost/127.0.0.1:35981 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-959958609-172.17.0.3-1733095688324 (Datanode Uuid 1b761c6e-5c73-49cf-aa4f-58c634bd83b5) service to localhost/127.0.0.1:35981 2024-12-01T23:29:15,233 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T23:29:15,236 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data3/current/BP-959958609-172.17.0.3-1733095688324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T23:29:15,236 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data4/current/BP-959958609-172.17.0.3-1733095688324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T23:29:15,240 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T23:29:15,265 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@278d3ff6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T23:29:15,268 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@b03447{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T23:29:15,268 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T23:29:15,268 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a482a5c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T23:29:15,269 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d8adca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/hadoop.log.dir/,STOPPED} 2024-12-01T23:29:15,279 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T23:29:15,279 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T23:29:15,279 WARN [BP-959958609-172.17.0.3-1733095688324 heartbeating to localhost/127.0.0.1:35981 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T23:29:15,280 WARN [BP-959958609-172.17.0.3-1733095688324 heartbeating to localhost/127.0.0.1:35981 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-959958609-172.17.0.3-1733095688324 (Datanode Uuid 4b04c2ae-3180-4679-9524-d641def164bb) service to localhost/127.0.0.1:35981 2024-12-01T23:29:15,281 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data1/current/BP-959958609-172.17.0.3-1733095688324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T23:29:15,282 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T23:29:15,283 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/cluster_3bba29f4-95ed-c976-4d7f-37ac1afc2f05/data/data2/current/BP-959958609-172.17.0.3-1733095688324 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T23:29:15,312 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@18f69b7e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-01T23:29:15,317 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@51bb6da6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T23:29:15,317 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T23:29:15,317 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e2d6db6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T23:29:15,317 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6626122{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f1c89bd-7899-7297-f765-96048a3fcd72/hadoop.log.dir/,STOPPED} 2024-12-01T23:29:15,350 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-01T23:29:15,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down