2024-12-09 18:53:18,031 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 18:53:18,045 main DEBUG Took 0.010905 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 18:53:18,045 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 18:53:18,045 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 18:53:18,046 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 18:53:18,047 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,054 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 18:53:18,065 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,067 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,068 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,069 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,070 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,071 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,072 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,072 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,074 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,074 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,075 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,075 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,076 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,076 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,077 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,078 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,079 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,080 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 18:53:18,080 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,081 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 18:53:18,083 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 18:53:18,084 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 18:53:18,086 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 18:53:18,087 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 18:53:18,088 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 18:53:18,089 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 18:53:18,101 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 18:53:18,104 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 18:53:18,106 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 18:53:18,106 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 18:53:18,107 main DEBUG createAppenders(={Console}) 2024-12-09 18:53:18,108 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-09 18:53:18,108 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 18:53:18,108 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-09 18:53:18,109 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 18:53:18,109 main DEBUG OutputStream closed 2024-12-09 18:53:18,110 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 18:53:18,110 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 18:53:18,110 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-09 18:53:18,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 18:53:18,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 18:53:18,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 18:53:18,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 18:53:18,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 18:53:18,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 18:53:18,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 18:53:18,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 18:53:18,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 18:53:18,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 18:53:18,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 18:53:18,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 18:53:18,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 18:53:18,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 18:53:18,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 18:53:18,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 18:53:18,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 18:53:18,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 18:53:18,187 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 18:53:18,188 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-09 18:53:18,188 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 18:53:18,189 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-09T18:53:18,412 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42 2024-12-09 18:53:18,415 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 18:53:18,415 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T18:53:18,424 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-09T18:53:18,431 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-09T18:53:18,455 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T18:53:18,501 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T18:53:18,502 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T18:53:18,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T18:53:18,534 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc, deleteOnExit=true 2024-12-09T18:53:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T18:53:18,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/test.cache.data in system properties and HBase conf 2024-12-09T18:53:18,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T18:53:18,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/hadoop.log.dir in system properties and HBase conf 2024-12-09T18:53:18,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T18:53:18,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T18:53:18,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T18:53:18,636 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T18:53:18,727 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T18:53:18,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T18:53:18,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T18:53:18,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T18:53:18,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T18:53:18,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T18:53:18,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T18:53:18,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T18:53:18,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T18:53:18,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T18:53:18,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/nfs.dump.dir in system properties and HBase conf 2024-12-09T18:53:18,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/java.io.tmpdir in system properties and HBase conf 2024-12-09T18:53:18,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T18:53:18,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T18:53:18,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T18:53:19,692 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T18:53:19,760 INFO [Time-limited test {}] log.Log(170): Logging initialized @2447ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T18:53:19,824 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T18:53:19,872 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T18:53:19,893 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T18:53:19,894 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T18:53:19,895 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T18:53:19,914 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T18:53:19,917 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e7ae814{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/hadoop.log.dir/,AVAILABLE} 2024-12-09T18:53:19,918 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3192a19b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T18:53:20,116 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5608f426{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/java.io.tmpdir/jetty-localhost-41265-hadoop-hdfs-3_4_1-tests_jar-_-any-3396908082076319636/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T18:53:20,121 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@63efd3a6{HTTP/1.1, (http/1.1)}{localhost:41265} 2024-12-09T18:53:20,122 INFO [Time-limited test {}] server.Server(415): Started @2809ms 2024-12-09T18:53:20,735 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T18:53:20,744 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T18:53:20,748 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T18:53:20,748 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T18:53:20,748 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T18:53:20,749 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@126e7def{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/hadoop.log.dir/,AVAILABLE} 2024-12-09T18:53:20,749 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fcc4f36{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T18:53:20,857 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@17d1a8d5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/java.io.tmpdir/jetty-localhost-39903-hadoop-hdfs-3_4_1-tests_jar-_-any-12738946293168644893/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T18:53:20,858 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4afdb36e{HTTP/1.1, (http/1.1)}{localhost:39903} 2024-12-09T18:53:20,858 INFO [Time-limited test {}] server.Server(415): Started @3545ms 2024-12-09T18:53:20,908 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T18:53:21,010 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T18:53:21,015 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T18:53:21,019 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T18:53:21,020 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T18:53:21,020 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T18:53:21,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@243ebd61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/hadoop.log.dir/,AVAILABLE} 2024-12-09T18:53:21,022 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f9065a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T18:53:21,117 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4464725f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/java.io.tmpdir/jetty-localhost-38757-hadoop-hdfs-3_4_1-tests_jar-_-any-136898764105215854/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T18:53:21,118 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11808715{HTTP/1.1, (http/1.1)}{localhost:38757} 2024-12-09T18:53:21,118 INFO [Time-limited test {}] server.Server(415): Started @3805ms 2024-12-09T18:53:21,120 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T18:53:21,153 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T18:53:21,158 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T18:53:21,159 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T18:53:21,160 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T18:53:21,160 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T18:53:21,160 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4bbcb0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/hadoop.log.dir/,AVAILABLE} 2024-12-09T18:53:21,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16c07ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T18:53:21,253 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26f8cb79{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/java.io.tmpdir/jetty-localhost-40639-hadoop-hdfs-3_4_1-tests_jar-_-any-4135504303808570343/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T18:53:21,254 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10705e91{HTTP/1.1, (http/1.1)}{localhost:40639} 2024-12-09T18:53:21,254 INFO [Time-limited test {}] server.Server(415): Started @3941ms 2024-12-09T18:53:21,256 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T18:53:22,198 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data2/current/BP-1882736132-172.17.0.2-1733770399228/current, will proceed with Du for space computation calculation, 2024-12-09T18:53:22,198 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data3/current/BP-1882736132-172.17.0.2-1733770399228/current, will proceed with Du for space computation calculation, 2024-12-09T18:53:22,198 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data4/current/BP-1882736132-172.17.0.2-1733770399228/current, will proceed with Du for space computation calculation, 2024-12-09T18:53:22,198 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data1/current/BP-1882736132-172.17.0.2-1733770399228/current, will proceed with Du for space computation calculation, 2024-12-09T18:53:22,234 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T18:53:22,234 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T18:53:22,273 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data5/current/BP-1882736132-172.17.0.2-1733770399228/current, will proceed with Du for space computation calculation, 2024-12-09T18:53:22,274 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data6/current/BP-1882736132-172.17.0.2-1733770399228/current, will proceed with Du for space computation calculation, 2024-12-09T18:53:22,281 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7321e10386b5490 with lease ID 0x273949bdd1c15f85: Processing first storage report for DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6 from datanode DatanodeRegistration(127.0.0.1:36059, datanodeUuid=379ba87a-21a3-409d-8cc1-0961b6a04a16, infoPort=44937, infoSecurePort=0, ipcPort=34513, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228) 2024-12-09T18:53:22,281 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7321e10386b5490 with lease ID 0x273949bdd1c15f85: from storage DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6 node DatanodeRegistration(127.0.0.1:36059, datanodeUuid=379ba87a-21a3-409d-8cc1-0961b6a04a16, infoPort=44937, infoSecurePort=0, ipcPort=34513, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T18:53:22,282 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d44880659ed958f with lease ID 0x273949bdd1c15f86: Processing first storage report for DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6 from datanode DatanodeRegistration(127.0.0.1:38029, datanodeUuid=b8ae8be4-88db-477b-9534-d9ab69a4f121, infoPort=36879, infoSecurePort=0, ipcPort=33619, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228) 2024-12-09T18:53:22,282 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d44880659ed958f with lease ID 0x273949bdd1c15f86: from storage DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6 node DatanodeRegistration(127.0.0.1:38029, datanodeUuid=b8ae8be4-88db-477b-9534-d9ab69a4f121, infoPort=36879, infoSecurePort=0, ipcPort=33619, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T18:53:22,282 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7321e10386b5490 with lease ID 0x273949bdd1c15f85: Processing first storage report for DS-eca4da7a-fb9b-41a1-a44f-cdc5ae8d2393 from datanode DatanodeRegistration(127.0.0.1:36059, datanodeUuid=379ba87a-21a3-409d-8cc1-0961b6a04a16, infoPort=44937, infoSecurePort=0, ipcPort=34513, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228) 2024-12-09T18:53:22,282 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7321e10386b5490 with lease ID 0x273949bdd1c15f85: from storage DS-eca4da7a-fb9b-41a1-a44f-cdc5ae8d2393 node DatanodeRegistration(127.0.0.1:36059, datanodeUuid=379ba87a-21a3-409d-8cc1-0961b6a04a16, infoPort=44937, infoSecurePort=0, ipcPort=34513, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T18:53:22,282 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d44880659ed958f with lease ID 0x273949bdd1c15f86: Processing first storage report for DS-0de95337-0ae9-4a43-8fb0-ead730d8918f from datanode DatanodeRegistration(127.0.0.1:38029, datanodeUuid=b8ae8be4-88db-477b-9534-d9ab69a4f121, infoPort=36879, infoSecurePort=0, ipcPort=33619, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228) 2024-12-09T18:53:22,283 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d44880659ed958f with lease ID 0x273949bdd1c15f86: from storage DS-0de95337-0ae9-4a43-8fb0-ead730d8918f node DatanodeRegistration(127.0.0.1:38029, datanodeUuid=b8ae8be4-88db-477b-9534-d9ab69a4f121, infoPort=36879, infoSecurePort=0, ipcPort=33619, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T18:53:22,295 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T18:53:22,299 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x498603fefe1c1d1a with lease ID 0x273949bdd1c15f87: Processing first storage report for DS-09c47033-c5b3-479d-880f-5fa06bff2e3a from datanode DatanodeRegistration(127.0.0.1:37407, datanodeUuid=d29cd452-4b12-4833-9e77-4b10cf55769f, infoPort=40071, infoSecurePort=0, ipcPort=34255, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228) 2024-12-09T18:53:22,299 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x498603fefe1c1d1a with lease ID 0x273949bdd1c15f87: from storage DS-09c47033-c5b3-479d-880f-5fa06bff2e3a node DatanodeRegistration(127.0.0.1:37407, datanodeUuid=d29cd452-4b12-4833-9e77-4b10cf55769f, infoPort=40071, infoSecurePort=0, ipcPort=34255, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T18:53:22,299 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x498603fefe1c1d1a with lease ID 0x273949bdd1c15f87: Processing first storage report for DS-6ececca1-72ae-4111-a760-cc281b3c1ff8 from datanode DatanodeRegistration(127.0.0.1:37407, datanodeUuid=d29cd452-4b12-4833-9e77-4b10cf55769f, infoPort=40071, infoSecurePort=0, ipcPort=34255, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228) 2024-12-09T18:53:22,299 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x498603fefe1c1d1a with lease ID 0x273949bdd1c15f87: from storage DS-6ececca1-72ae-4111-a760-cc281b3c1ff8 node DatanodeRegistration(127.0.0.1:37407, datanodeUuid=d29cd452-4b12-4833-9e77-4b10cf55769f, infoPort=40071, infoSecurePort=0, ipcPort=34255, storageInfo=lv=-57;cid=testClusterID;nsid=1713810906;c=1733770399228), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T18:53:22,381 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42 2024-12-09T18:53:22,440 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/zookeeper_0, clientPort=52975, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T18:53:22,457 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52975 2024-12-09T18:53:22,466 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:22,469 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:22,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741825_1001 (size=7) 2024-12-09T18:53:22,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741825_1001 (size=7) 2024-12-09T18:53:22,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741825_1001 (size=7) 2024-12-09T18:53:23,055 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8 with version=8 2024-12-09T18:53:23,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/hbase-staging 2024-12-09T18:53:23,343 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a437f8b9ba7d:0 server-side Connection retries=45 2024-12-09T18:53:23,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:23,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:23,356 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T18:53:23,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:23,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T18:53:23,480 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T18:53:23,533 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T18:53:23,544 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T18:53:23,548 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T18:53:23,574 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 77124 (auto-detected) 2024-12-09T18:53:23,575 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T18:53:23,592 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35375 2024-12-09T18:53:23,610 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35375 connecting to ZooKeeper ensemble=127.0.0.1:52975 2024-12-09T18:53:23,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:353750x0, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T18:53:23,713 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35375-0x1000c23f6fa0000 connected 2024-12-09T18:53:23,806 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:23,810 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:23,821 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T18:53:23,826 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8, hbase.cluster.distributed=false 2024-12-09T18:53:23,852 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T18:53:23,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35375 2024-12-09T18:53:23,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35375 2024-12-09T18:53:23,859 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35375 2024-12-09T18:53:23,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35375 2024-12-09T18:53:23,860 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35375 2024-12-09T18:53:23,957 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a437f8b9ba7d:0 server-side Connection retries=45 2024-12-09T18:53:23,958 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:23,959 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:23,959 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T18:53:23,959 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:23,959 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T18:53:23,962 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T18:53:23,964 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T18:53:23,965 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35449 2024-12-09T18:53:23,966 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35449 connecting to ZooKeeper ensemble=127.0.0.1:52975 2024-12-09T18:53:23,967 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:23,974 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:23,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354490x0, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T18:53:23,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:354490x0, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T18:53:23,999 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35449-0x1000c23f6fa0001 connected 2024-12-09T18:53:24,000 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T18:53:24,007 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T18:53:24,010 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T18:53:24,015 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T18:53:24,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35449 2024-12-09T18:53:24,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35449 2024-12-09T18:53:24,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35449 2024-12-09T18:53:24,017 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35449 2024-12-09T18:53:24,017 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35449 2024-12-09T18:53:24,032 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a437f8b9ba7d:0 server-side Connection retries=45 2024-12-09T18:53:24,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:24,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:24,032 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T18:53:24,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:24,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T18:53:24,033 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T18:53:24,033 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T18:53:24,033 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46657 2024-12-09T18:53:24,035 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46657 connecting to ZooKeeper ensemble=127.0.0.1:52975 2024-12-09T18:53:24,036 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:24,039 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:24,054 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466570x0, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T18:53:24,055 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46657-0x1000c23f6fa0002 connected 2024-12-09T18:53:24,055 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:466570x0, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T18:53:24,056 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T18:53:24,057 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T18:53:24,058 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T18:53:24,060 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T18:53:24,061 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46657 2024-12-09T18:53:24,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46657 2024-12-09T18:53:24,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46657 2024-12-09T18:53:24,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46657 2024-12-09T18:53:24,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46657 2024-12-09T18:53:24,084 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a437f8b9ba7d:0 server-side Connection retries=45 2024-12-09T18:53:24,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:24,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:24,085 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T18:53:24,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T18:53:24,085 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T18:53:24,085 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T18:53:24,086 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T18:53:24,087 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37501 2024-12-09T18:53:24,088 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37501 connecting to ZooKeeper ensemble=127.0.0.1:52975 2024-12-09T18:53:24,090 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:24,093 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:24,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:375010x0, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T18:53:24,105 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:375010x0, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T18:53:24,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37501-0x1000c23f6fa0003 connected 2024-12-09T18:53:24,106 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T18:53:24,108 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T18:53:24,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T18:53:24,111 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T18:53:24,114 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37501 2024-12-09T18:53:24,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37501 2024-12-09T18:53:24,121 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37501 2024-12-09T18:53:24,122 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37501 2024-12-09T18:53:24,122 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37501 2024-12-09T18:53:24,134 DEBUG [M:0;a437f8b9ba7d:35375 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a437f8b9ba7d:35375 2024-12-09T18:53:24,135 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a437f8b9ba7d,35375,1733770403171 2024-12-09T18:53:24,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T18:53:24,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T18:53:24,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T18:53:24,146 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T18:53:24,149 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a437f8b9ba7d,35375,1733770403171 2024-12-09T18:53:24,183 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T18:53:24,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T18:53:24,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:24,184 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:24,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:24,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T18:53:24,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:24,185 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T18:53:24,187 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a437f8b9ba7d,35375,1733770403171 from backup master directory 2024-12-09T18:53:24,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a437f8b9ba7d,35375,1733770403171 2024-12-09T18:53:24,196 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T18:53:24,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T18:53:24,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T18:53:24,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T18:53:24,198 WARN [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T18:53:24,198 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a437f8b9ba7d,35375,1733770403171 2024-12-09T18:53:24,201 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T18:53:24,202 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T18:53:24,279 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/hbase.id] with ID: 09f08c9b-bbeb-413c-9bb5-a0da44c958b7 2024-12-09T18:53:24,279 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/.tmp/hbase.id 2024-12-09T18:53:24,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741826_1002 (size=42) 2024-12-09T18:53:24,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741826_1002 (size=42) 2024-12-09T18:53:24,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741826_1002 (size=42) 2024-12-09T18:53:24,305 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/.tmp/hbase.id]:[hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/hbase.id] 2024-12-09T18:53:24,351 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T18:53:24,355 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T18:53:24,374 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-09T18:53:24,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:24,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:24,384 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:24,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:24,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741827_1003 (size=196) 2024-12-09T18:53:24,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741827_1003 (size=196) 2024-12-09T18:53:24,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741827_1003 (size=196) 2024-12-09T18:53:24,419 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T18:53:24,421 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T18:53:24,426 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:24,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741828_1004 (size=1189) 2024-12-09T18:53:24,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741828_1004 (size=1189) 2024-12-09T18:53:24,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741828_1004 (size=1189) 2024-12-09T18:53:24,487 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store 2024-12-09T18:53:24,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741829_1005 (size=34) 2024-12-09T18:53:24,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741829_1005 (size=34) 2024-12-09T18:53:24,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741829_1005 (size=34) 2024-12-09T18:53:24,529 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T18:53:24,534 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:24,536 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T18:53:24,537 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T18:53:24,537 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T18:53:24,540 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T18:53:24,540 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T18:53:24,541 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T18:53:24,543 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733770404536Disabling compacts and flushes for region at 1733770404536Disabling writes for close at 1733770404540 (+4 ms)Writing region close event to WAL at 1733770404541 (+1 ms)Closed at 1733770404541 2024-12-09T18:53:24,547 WARN [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/.initializing 2024-12-09T18:53:24,548 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/WALs/a437f8b9ba7d,35375,1733770403171 2024-12-09T18:53:24,560 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:24,577 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a437f8b9ba7d%2C35375%2C1733770403171, suffix=, logDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/WALs/a437f8b9ba7d,35375,1733770403171, archiveDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/oldWALs, maxLogs=10 2024-12-09T18:53:24,620 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/WALs/a437f8b9ba7d,35375,1733770403171/a437f8b9ba7d%2C35375%2C1733770403171.1733770404584, exclude list is [], retry=0 2024-12-09T18:53:24,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:24,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:24,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:24,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:24,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T18:53:24,690 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-09T18:53:24,773 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/WALs/a437f8b9ba7d,35375,1733770403171/a437f8b9ba7d%2C35375%2C1733770403171.1733770404584 2024-12-09T18:53:24,774 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071)] 2024-12-09T18:53:24,775 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:24,775 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:24,779 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,780 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,824 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,857 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T18:53:24,863 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:24,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:24,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,872 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T18:53:24,872 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:24,874 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:24,874 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,879 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T18:53:24,879 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:24,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:24,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T18:53:24,887 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:24,889 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:24,889 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,894 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,897 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,905 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,906 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,912 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T18:53:24,919 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T18:53:24,926 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:24,928 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71283905, jitterRate=0.06221295893192291}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T18:53:24,936 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733770404796Initializing all the Stores at 1733770404798 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733770404798Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770404799 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770404799Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770404799Cleaning up temporary data from old regions at 1733770404906 (+107 ms)Region opened successfully at 1733770404936 (+30 ms) 2024-12-09T18:53:24,938 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T18:53:24,982 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd4fd8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a437f8b9ba7d/172.17.0.2:0 2024-12-09T18:53:25,020 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T18:53:25,032 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T18:53:25,032 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T18:53:25,035 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T18:53:25,037 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T18:53:25,042 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T18:53:25,042 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T18:53:25,073 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T18:53:25,082 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T18:53:25,096 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T18:53:25,098 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T18:53:25,100 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T18:53:25,108 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T18:53:25,111 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T18:53:25,115 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T18:53:25,125 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T18:53:25,127 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T18:53:25,137 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T18:53:25,154 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T18:53:25,162 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T18:53:25,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T18:53:25,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T18:53:25,175 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T18:53:25,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T18:53:25,175 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,179 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a437f8b9ba7d,35375,1733770403171, sessionid=0x1000c23f6fa0000, setting cluster-up flag (Was=false) 2024-12-09T18:53:25,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,205 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,229 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T18:53:25,232 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a437f8b9ba7d,35375,1733770403171 2024-12-09T18:53:25,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,254 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:25,279 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T18:53:25,281 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a437f8b9ba7d,35375,1733770403171 2024-12-09T18:53:25,286 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T18:53:25,328 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(746): ClusterId : 09f08c9b-bbeb-413c-9bb5-a0da44c958b7 2024-12-09T18:53:25,328 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(746): ClusterId : 09f08c9b-bbeb-413c-9bb5-a0da44c958b7 2024-12-09T18:53:25,328 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(746): ClusterId : 09f08c9b-bbeb-413c-9bb5-a0da44c958b7 2024-12-09T18:53:25,336 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T18:53:25,336 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T18:53:25,336 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T18:53:25,356 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T18:53:25,356 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T18:53:25,358 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T18:53:25,358 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T18:53:25,365 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T18:53:25,367 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T18:53:25,367 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T18:53:25,374 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T18:53:25,381 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T18:53:25,383 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T18:53:25,384 DEBUG [RS:2;a437f8b9ba7d:37501 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a12b30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a437f8b9ba7d/172.17.0.2:0 2024-12-09T18:53:25,384 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T18:53:25,385 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T18:53:25,385 DEBUG [RS:1;a437f8b9ba7d:46657 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad68d9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a437f8b9ba7d/172.17.0.2:0 2024-12-09T18:53:25,385 DEBUG [RS:0;a437f8b9ba7d:35449 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c23098e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a437f8b9ba7d/172.17.0.2:0 2024-12-09T18:53:25,389 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a437f8b9ba7d,35375,1733770403171 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T18:53:25,398 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a437f8b9ba7d:0, corePoolSize=5, maxPoolSize=5 2024-12-09T18:53:25,398 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a437f8b9ba7d:0, corePoolSize=5, maxPoolSize=5 2024-12-09T18:53:25,399 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a437f8b9ba7d:0, corePoolSize=5, maxPoolSize=5 2024-12-09T18:53:25,399 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a437f8b9ba7d:0, corePoolSize=5, maxPoolSize=5 2024-12-09T18:53:25,399 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a437f8b9ba7d:0, corePoolSize=10, maxPoolSize=10 2024-12-09T18:53:25,399 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,399 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a437f8b9ba7d:0, corePoolSize=2, maxPoolSize=2 2024-12-09T18:53:25,399 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,404 DEBUG [RS:0;a437f8b9ba7d:35449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a437f8b9ba7d:35449 2024-12-09T18:53:25,404 DEBUG [RS:1;a437f8b9ba7d:46657 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a437f8b9ba7d:46657 2024-12-09T18:53:25,404 DEBUG [RS:2;a437f8b9ba7d:37501 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a437f8b9ba7d:37501 2024-12-09T18:53:25,409 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T18:53:25,409 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T18:53:25,409 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T18:53:25,409 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T18:53:25,409 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T18:53:25,409 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T18:53:25,409 DEBUG [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T18:53:25,409 DEBUG [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T18:53:25,409 DEBUG [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T18:53:25,412 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(2659): reportForDuty to master=a437f8b9ba7d,35375,1733770403171 with port=46657, startcode=1733770404031 2024-12-09T18:53:25,412 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(2659): reportForDuty to master=a437f8b9ba7d,35375,1733770403171 with port=35449, startcode=1733770403924 2024-12-09T18:53:25,414 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(2659): reportForDuty to master=a437f8b9ba7d,35375,1733770403171 with port=37501, startcode=1733770404084 2024-12-09T18:53:25,417 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T18:53:25,418 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T18:53:25,424 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:25,424 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733770435424 2024-12-09T18:53:25,425 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T18:53:25,426 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T18:53:25,426 DEBUG [RS:0;a437f8b9ba7d:35449 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T18:53:25,426 DEBUG [RS:2;a437f8b9ba7d:37501 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T18:53:25,428 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T18:53:25,427 DEBUG [RS:1;a437f8b9ba7d:46657 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T18:53:25,432 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T18:53:25,432 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T18:53:25,432 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T18:53:25,434 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T18:53:25,439 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,452 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T18:53:25,453 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T18:53:25,454 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T18:53:25,457 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T18:53:25,488 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T18:53:25,491 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.large.0-1733770405489,5,FailOnTimeoutGroup] 2024-12-09T18:53:25,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741831_1007 (size=1321) 2024-12-09T18:53:25,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741831_1007 (size=1321) 2024-12-09T18:53:25,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741831_1007 (size=1321) 2024-12-09T18:53:25,495 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.small.0-1733770405491,5,FailOnTimeoutGroup] 2024-12-09T18:53:25,495 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,495 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T18:53:25,498 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T18:53:25,498 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8 2024-12-09T18:53:25,499 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,499 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,510 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52023, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T18:53:25,510 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53469, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T18:53:25,510 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32899, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T18:53:25,517 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35375 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:25,520 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35375 {}] master.ServerManager(517): Registering regionserver=a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:25,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741832_1008 (size=32) 2024-12-09T18:53:25,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741832_1008 (size=32) 2024-12-09T18:53:25,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741832_1008 (size=32) 2024-12-09T18:53:25,536 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35375 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:25,536 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35375 {}] master.ServerManager(517): Registering regionserver=a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:25,538 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:25,538 DEBUG [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8 2024-12-09T18:53:25,539 DEBUG [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41577 2024-12-09T18:53:25,539 DEBUG [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T18:53:25,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T18:53:25,545 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T18:53:25,545 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:25,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:25,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T18:53:25,552 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T18:53:25,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:25,553 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35375 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:25,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:25,553 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35375 {}] master.ServerManager(517): Registering regionserver=a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:25,554 DEBUG [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8 2024-12-09T18:53:25,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T18:53:25,554 DEBUG [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41577 2024-12-09T18:53:25,554 DEBUG [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T18:53:25,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T18:53:25,555 DEBUG [RS:1;a437f8b9ba7d:46657 {}] zookeeper.ZKUtil(111): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:25,555 WARN [RS:1;a437f8b9ba7d:46657 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T18:53:25,555 INFO [RS:1;a437f8b9ba7d:46657 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:25,555 DEBUG [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:25,558 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T18:53:25,558 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:25,559 DEBUG [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8 2024-12-09T18:53:25,559 DEBUG [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41577 2024-12-09T18:53:25,559 DEBUG [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T18:53:25,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:25,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T18:53:25,563 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T18:53:25,563 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:25,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:25,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T18:53:25,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740 2024-12-09T18:53:25,568 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740 2024-12-09T18:53:25,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T18:53:25,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T18:53:25,573 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T18:53:25,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T18:53:25,582 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:25,583 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70241931, jitterRate=0.046686336398124695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T18:53:25,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733770405538Initializing all the Stores at 1733770405540 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733770405540Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733770405541 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770405541Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733770405541Cleaning up temporary data from old regions at 1733770405572 (+31 ms)Region opened successfully at 1733770405585 (+13 ms) 2024-12-09T18:53:25,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T18:53:25,586 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T18:53:25,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T18:53:25,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T18:53:25,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T18:53:25,588 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T18:53:25,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733770405586Disabling compacts and flushes for region at 1733770405586Disabling writes for close at 1733770405586Writing region close event to WAL at 1733770405587 (+1 ms)Closed at 1733770405587 2024-12-09T18:53:25,591 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T18:53:25,591 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T18:53:25,596 DEBUG [RS:0;a437f8b9ba7d:35449 {}] zookeeper.ZKUtil(111): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:25,598 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a437f8b9ba7d,46657,1733770404031] 2024-12-09T18:53:25,598 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a437f8b9ba7d,35449,1733770403924] 2024-12-09T18:53:25,598 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a437f8b9ba7d,37501,1733770404084] 2024-12-09T18:53:25,596 WARN [RS:0;a437f8b9ba7d:35449 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T18:53:25,599 INFO [RS:0;a437f8b9ba7d:35449 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:25,599 DEBUG [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:25,600 DEBUG [RS:2;a437f8b9ba7d:37501 {}] zookeeper.ZKUtil(111): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:25,600 WARN [RS:2;a437f8b9ba7d:37501 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T18:53:25,600 INFO [RS:2;a437f8b9ba7d:37501 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:25,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T18:53:25,600 DEBUG [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:25,616 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T18:53:25,620 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T18:53:25,626 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T18:53:25,626 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T18:53:25,626 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T18:53:25,638 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T18:53:25,639 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T18:53:25,642 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T18:53:25,648 INFO [RS:1;a437f8b9ba7d:46657 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T18:53:25,648 INFO [RS:0;a437f8b9ba7d:35449 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T18:53:25,648 INFO [RS:2;a437f8b9ba7d:37501 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T18:53:25,648 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,648 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,648 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,649 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T18:53:25,655 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T18:53:25,657 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T18:53:25,657 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T18:53:25,658 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T18:53:25,659 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T18:53:25,659 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,659 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,660 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,660 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,660 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0, corePoolSize=2, maxPoolSize=2 2024-12-09T18:53:25,661 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0, corePoolSize=2, maxPoolSize=2 2024-12-09T18:53:25,661 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0, corePoolSize=2, maxPoolSize=2 2024-12-09T18:53:25,661 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,661 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,662 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,662 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a437f8b9ba7d:0, corePoolSize=1, maxPoolSize=1 2024-12-09T18:53:25,662 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a437f8b9ba7d:0, corePoolSize=3, maxPoolSize=3 2024-12-09T18:53:25,662 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a437f8b9ba7d:0, corePoolSize=3, maxPoolSize=3 2024-12-09T18:53:25,662 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a437f8b9ba7d:0, corePoolSize=3, maxPoolSize=3 2024-12-09T18:53:25,662 DEBUG [RS:2;a437f8b9ba7d:37501 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0, corePoolSize=3, maxPoolSize=3 2024-12-09T18:53:25,662 DEBUG [RS:1;a437f8b9ba7d:46657 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0, corePoolSize=3, maxPoolSize=3 2024-12-09T18:53:25,662 DEBUG [RS:0;a437f8b9ba7d:35449 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a437f8b9ba7d:0, corePoolSize=3, maxPoolSize=3 2024-12-09T18:53:25,668 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,668 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,668 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,668 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,669 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,669 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,46657,1733770404031-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T18:53:25,670 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,670 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,670 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,670 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,670 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,671 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,37501,1733770404084-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T18:53:25,675 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,676 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,676 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,676 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,677 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,677 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,35449,1733770403924-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T18:53:25,695 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T18:53:25,695 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T18:53:25,697 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,46657,1733770404031-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,697 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,37501,1733770404084-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,698 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,698 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,698 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.Replication(171): a437f8b9ba7d,37501,1733770404084 started 2024-12-09T18:53:25,698 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.Replication(171): a437f8b9ba7d,46657,1733770404031 started 2024-12-09T18:53:25,699 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T18:53:25,699 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,35449,1733770403924-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,699 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,700 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.Replication(171): a437f8b9ba7d,35449,1733770403924 started 2024-12-09T18:53:25,724 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,724 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,724 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(1482): Serving as a437f8b9ba7d,37501,1733770404084, RpcServer on a437f8b9ba7d/172.17.0.2:37501, sessionid=0x1000c23f6fa0003 2024-12-09T18:53:25,724 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(1482): Serving as a437f8b9ba7d,35449,1733770403924, RpcServer on a437f8b9ba7d/172.17.0.2:35449, sessionid=0x1000c23f6fa0001 2024-12-09T18:53:25,725 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T18:53:25,725 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T18:53:25,726 DEBUG [RS:0;a437f8b9ba7d:35449 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:25,726 DEBUG [RS:2;a437f8b9ba7d:37501 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:25,726 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a437f8b9ba7d,35449,1733770403924' 2024-12-09T18:53:25,726 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a437f8b9ba7d,37501,1733770404084' 2024-12-09T18:53:25,726 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T18:53:25,726 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T18:53:25,727 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T18:53:25,727 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T18:53:25,728 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T18:53:25,728 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T18:53:25,729 DEBUG [RS:0;a437f8b9ba7d:35449 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:25,729 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a437f8b9ba7d,35449,1733770403924' 2024-12-09T18:53:25,729 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T18:53:25,729 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T18:53:25,729 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T18:53:25,729 DEBUG [RS:2;a437f8b9ba7d:37501 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:25,729 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a437f8b9ba7d,37501,1733770404084' 2024-12-09T18:53:25,729 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T18:53:25,730 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:25,730 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T18:53:25,730 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(1482): Serving as a437f8b9ba7d,46657,1733770404031, RpcServer on a437f8b9ba7d/172.17.0.2:46657, sessionid=0x1000c23f6fa0002 2024-12-09T18:53:25,730 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T18:53:25,730 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T18:53:25,730 DEBUG [RS:1;a437f8b9ba7d:46657 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:25,730 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a437f8b9ba7d,46657,1733770404031' 2024-12-09T18:53:25,731 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T18:53:25,731 DEBUG [RS:0;a437f8b9ba7d:35449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T18:53:25,731 INFO [RS:0;a437f8b9ba7d:35449 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T18:53:25,731 INFO [RS:0;a437f8b9ba7d:35449 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T18:53:25,731 DEBUG [RS:2;a437f8b9ba7d:37501 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T18:53:25,731 INFO [RS:2;a437f8b9ba7d:37501 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T18:53:25,731 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T18:53:25,731 INFO [RS:2;a437f8b9ba7d:37501 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T18:53:25,732 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T18:53:25,732 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T18:53:25,732 DEBUG [RS:1;a437f8b9ba7d:46657 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:25,732 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a437f8b9ba7d,46657,1733770404031' 2024-12-09T18:53:25,732 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T18:53:25,733 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T18:53:25,734 DEBUG [RS:1;a437f8b9ba7d:46657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T18:53:25,734 INFO [RS:1;a437f8b9ba7d:46657 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T18:53:25,734 INFO [RS:1;a437f8b9ba7d:46657 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T18:53:25,771 WARN [a437f8b9ba7d:35375 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T18:53:25,836 INFO [RS:0;a437f8b9ba7d:35449 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:25,837 INFO [RS:2;a437f8b9ba7d:37501 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:25,837 INFO [RS:1;a437f8b9ba7d:46657 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:25,840 INFO [RS:2;a437f8b9ba7d:37501 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a437f8b9ba7d%2C37501%2C1733770404084, suffix=, logDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,37501,1733770404084, archiveDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/oldWALs, maxLogs=32 2024-12-09T18:53:25,840 INFO [RS:1;a437f8b9ba7d:46657 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a437f8b9ba7d%2C46657%2C1733770404031, suffix=, logDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031, archiveDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/oldWALs, maxLogs=32 2024-12-09T18:53:25,840 INFO [RS:0;a437f8b9ba7d:35449 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a437f8b9ba7d%2C35449%2C1733770403924, suffix=, logDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,35449,1733770403924, archiveDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/oldWALs, maxLogs=32 2024-12-09T18:53:25,862 DEBUG [RS:0;a437f8b9ba7d:35449 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,35449,1733770403924/a437f8b9ba7d%2C35449%2C1733770403924.1733770405847, exclude list is [], retry=0 2024-12-09T18:53:25,862 DEBUG [RS:2;a437f8b9ba7d:37501 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,37501,1733770404084/a437f8b9ba7d%2C37501%2C1733770404084.1733770405845, exclude list is [], retry=0 2024-12-09T18:53:25,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:25,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:25,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:25,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:25,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:25,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:25,872 DEBUG [RS:1;a437f8b9ba7d:46657 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845, exclude list is [], retry=0 2024-12-09T18:53:25,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:25,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:25,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:25,927 INFO [RS:0;a437f8b9ba7d:35449 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,35449,1733770403924/a437f8b9ba7d%2C35449%2C1733770403924.1733770405847 2024-12-09T18:53:25,927 INFO [RS:2;a437f8b9ba7d:37501 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,37501,1733770404084/a437f8b9ba7d%2C37501%2C1733770404084.1733770405845 2024-12-09T18:53:25,929 DEBUG [RS:2;a437f8b9ba7d:37501 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:25,931 DEBUG [RS:0;a437f8b9ba7d:35449 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:25,936 INFO [RS:1;a437f8b9ba7d:46657 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845 2024-12-09T18:53:25,937 DEBUG [RS:1;a437f8b9ba7d:46657 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:36879:36879)] 2024-12-09T18:53:26,024 DEBUG [a437f8b9ba7d:35375 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-09T18:53:26,032 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(204): Hosts are {a437f8b9ba7d=0} racks are {/default-rack=0} 2024-12-09T18:53:26,039 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T18:53:26,039 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T18:53:26,039 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T18:53:26,039 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T18:53:26,039 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T18:53:26,039 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T18:53:26,039 INFO [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T18:53:26,039 INFO [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T18:53:26,039 INFO [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T18:53:26,039 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T18:53:26,045 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:26,049 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a437f8b9ba7d,46657,1733770404031, state=OPENING 2024-12-09T18:53:26,079 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T18:53:26,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:26,124 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:26,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:26,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:53:26,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:26,127 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:26,127 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:26,127 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:26,130 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T18:53:26,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a437f8b9ba7d,46657,1733770404031}] 2024-12-09T18:53:26,306 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T18:53:26,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47547, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T18:53:26,322 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T18:53:26,322 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:26,323 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T18:53:26,326 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a437f8b9ba7d%2C46657%2C1733770404031.meta, suffix=.meta, logDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031, archiveDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/oldWALs, maxLogs=32 2024-12-09T18:53:26,341 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta, exclude list is [], retry=0 2024-12-09T18:53:26,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:26,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:26,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:26,377 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta 2024-12-09T18:53:26,377 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:26,378 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:26,379 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T18:53:26,382 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T18:53:26,388 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T18:53:26,394 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T18:53:26,394 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:26,394 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T18:53:26,395 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T18:53:26,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T18:53:26,400 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T18:53:26,401 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:26,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:26,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T18:53:26,404 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T18:53:26,404 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:26,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:26,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T18:53:26,407 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T18:53:26,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:26,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:26,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T18:53:26,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T18:53:26,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:26,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:26,412 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T18:53:26,414 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740 2024-12-09T18:53:26,417 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740 2024-12-09T18:53:26,420 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T18:53:26,420 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T18:53:26,421 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T18:53:26,424 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T18:53:26,426 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61954056, jitterRate=-0.07681262493133545}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T18:53:26,427 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T18:53:26,428 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733770406395Writing region info on filesystem at 1733770406395Initializing all the Stores at 1733770406398 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733770406398Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733770406398Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770406398Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733770406398Cleaning up temporary data from old regions at 1733770406420 (+22 ms)Running coprocessor post-open hooks at 1733770406427 (+7 ms)Region opened successfully at 1733770406428 (+1 ms) 2024-12-09T18:53:26,435 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733770406299 2024-12-09T18:53:26,447 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T18:53:26,447 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T18:53:26,449 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:26,452 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a437f8b9ba7d,46657,1733770404031, state=OPEN 2024-12-09T18:53:26,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T18:53:26,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T18:53:26,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T18:53:26,467 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T18:53:26,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:26,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:26,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:26,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:26,467 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:26,478 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T18:53:26,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a437f8b9ba7d,46657,1733770404031 in 336 msec 2024-12-09T18:53:26,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T18:53:26,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 882 msec 2024-12-09T18:53:26,495 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T18:53:26,495 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T18:53:26,522 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T18:53:26,524 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1] 2024-12-09T18:53:26,553 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T18:53:26,555 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55393, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T18:53:26,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2600 sec 2024-12-09T18:53:26,578 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733770406578, completionTime=-1 2024-12-09T18:53:26,582 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-09T18:53:26,582 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T18:53:26,641 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-09T18:53:26,641 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733770466641 2024-12-09T18:53:26,641 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733770526641 2024-12-09T18:53:26,641 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 58 msec 2024-12-09T18:53:26,643 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T18:53:26,671 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,35375,1733770403171-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:26,672 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,35375,1733770403171-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:26,672 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,35375,1733770403171-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:26,674 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a437f8b9ba7d:35375, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:26,674 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:26,675 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:26,684 DEBUG [master/a437f8b9ba7d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T18:53:26,714 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.515sec 2024-12-09T18:53:26,716 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T18:53:26,718 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T18:53:26,720 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T18:53:26,721 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T18:53:26,721 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T18:53:26,723 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,35375,1733770403171-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T18:53:26,723 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,35375,1733770403171-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T18:53:26,733 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T18:53:26,734 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T18:53:26,735 INFO [master/a437f8b9ba7d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a437f8b9ba7d,35375,1733770403171-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:26,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@372ca112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T18:53:26,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a437f8b9ba7d,35375,-1 for getting cluster id 2024-12-09T18:53:26,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T18:53:26,765 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f08c9b-bbeb-413c-9bb5-a0da44c958b7' 2024-12-09T18:53:26,768 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T18:53:26,768 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f08c9b-bbeb-413c-9bb5-a0da44c958b7" 2024-12-09T18:53:26,769 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f7e9f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T18:53:26,769 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a437f8b9ba7d,35375,-1] 2024-12-09T18:53:26,772 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T18:53:26,775 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T18:53:26,776 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T18:53:26,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@759e1daf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T18:53:26,780 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T18:53:26,788 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1] 2024-12-09T18:53:26,789 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T18:53:26,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T18:53:26,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a437f8b9ba7d,35375,1733770403171 2024-12-09T18:53:26,817 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:41577/hbase 2024-12-09T18:53:26,837 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=369, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=190, ProcessCount=11, AvailableMemoryMB=3584 2024-12-09T18:53:26,863 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:26,867 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:26,868 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:26,873 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-91423815, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/hregion-91423815, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:26,893 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-91423815/hregion-91423815.1733770406874, exclude list is [], retry=0 2024-12-09T18:53:26,900 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:26,902 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:26,904 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:26,939 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-91423815/hregion-91423815.1733770406874 2024-12-09T18:53:26,940 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071)] 2024-12-09T18:53:26,940 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 23bfcc9a8cb95c05d9e83db5761e75c0, NAME => 'testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/hbase 2024-12-09T18:53:26,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741838_1014 (size=64) 2024-12-09T18:53:26,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741838_1014 (size=64) 2024-12-09T18:53:26,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741838_1014 (size=64) 2024-12-09T18:53:26,957 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:26,958 INFO [StoreOpener-23bfcc9a8cb95c05d9e83db5761e75c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:26,961 INFO [StoreOpener-23bfcc9a8cb95c05d9e83db5761e75c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23bfcc9a8cb95c05d9e83db5761e75c0 columnFamilyName a 2024-12-09T18:53:26,961 DEBUG [StoreOpener-23bfcc9a8cb95c05d9e83db5761e75c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:26,962 INFO [StoreOpener-23bfcc9a8cb95c05d9e83db5761e75c0-1 {}] regionserver.HStore(327): Store=23bfcc9a8cb95c05d9e83db5761e75c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:26,963 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:26,964 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:26,965 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:26,965 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:26,966 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:26,968 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:26,972 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:26,973 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 23bfcc9a8cb95c05d9e83db5761e75c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69481402, jitterRate=0.03535357117652893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T18:53:26,974 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 23bfcc9a8cb95c05d9e83db5761e75c0: Writing region info on filesystem at 1733770406957Initializing all the Stores at 1733770406958 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770406958Cleaning up temporary data from old regions at 1733770406966 (+8 ms)Region opened successfully at 1733770406974 (+8 ms) 2024-12-09T18:53:26,974 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 23bfcc9a8cb95c05d9e83db5761e75c0, disabling compactions & flushes 2024-12-09T18:53:26,974 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0. 2024-12-09T18:53:26,974 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0. 2024-12-09T18:53:26,974 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0. after waiting 0 ms 2024-12-09T18:53:26,974 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0. 2024-12-09T18:53:26,975 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0. 2024-12-09T18:53:26,975 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 23bfcc9a8cb95c05d9e83db5761e75c0: Waiting for close lock at 1733770406974Disabling compacts and flushes for region at 1733770406974Disabling writes for close at 1733770406974Writing region close event to WAL at 1733770406975 (+1 ms)Closed at 1733770406975 2024-12-09T18:53:26,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741837_1013 (size=175) 2024-12-09T18:53:26,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741837_1013 (size=175) 2024-12-09T18:53:26,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741837_1013 (size=175) 2024-12-09T18:53:26,986 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:26,986 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-91423815:(num 1733770406874) 2024-12-09T18:53:26,988 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T18:53:27,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741839_1015 (size=469) 2024-12-09T18:53:27,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741839_1015 (size=469) 2024-12-09T18:53:27,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741839_1015 (size=469) 2024-12-09T18:53:27,034 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T18:53:27,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741840_1016 (size=322) 2024-12-09T18:53:27,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741840_1016 (size=322) 2024-12-09T18:53:27,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741840_1016 (size=322) 2024-12-09T18:53:27,094 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-1, size=469 (469bytes) 2024-12-09T18:53:27,095 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-09T18:53:27,095 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-09T18:53:27,095 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-1 2024-12-09T18:53:27,100 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-1 after 3ms 2024-12-09T18:53:27,129 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:27,130 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-1 took 37ms 2024-12-09T18:53:27,135 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-1 so closing down 2024-12-09T18:53:27,135 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:27,139 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T18:53:27,162 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T18:53:27,163 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:27,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741841_1017 (size=469) 2024-12-09T18:53:27,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741841_1017 (size=469) 2024-12-09T18:53:27,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741841_1017 (size=469) 2024-12-09T18:53:27,175 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T18:53:27,178 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000002 2024-12-09T18:53:27,181 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-09T18:53:27,182 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-1, journal: Splitting hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-1, size=469 (469bytes) at 1733770407094Finishing writing output for hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-1 so closing down at 1733770407135 (+41 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000001-wal-1.temp at 1733770407162 (+27 ms)3 split writer threads finished at 1733770407163 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733770407175 (+12 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000002 at 1733770407178 (+3 ms)Processed 2 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733770407181 (+3 ms) 2024-12-09T18:53:27,195 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-2, size=322 (322bytes) 2024-12-09T18:53:27,195 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-2 2024-12-09T18:53:27,196 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-2 after 1ms 2024-12-09T18:53:27,224 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:27,224 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-2 took 29ms 2024-12-09T18:53:27,227 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-2 so closing down 2024-12-09T18:53:27,227 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:27,230 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T18:53:27,256 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T18:53:27,256 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:27,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741842_1018 (size=322) 2024-12-09T18:53:27,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741842_1018 (size=322) 2024-12-09T18:53:27,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741842_1018 (size=322) 2024-12-09T18:53:27,271 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T18:53:27,304 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:27,307 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-09T18:53:27,309 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 84 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-09T18:53:27,309 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-2, journal: Splitting hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-2, size=322 (322bytes) at 1733770407195Finishing writing output for hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-2 so closing down at 1733770407227 (+32 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000002-wal-2.temp at 1733770407256 (+29 ms)3 split writer threads finished at 1733770407256Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733770407271 (+15 ms)Processed 1 edits across 1 Regions in 84 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733770407309 (+38 ms) 2024-12-09T18:53:27,309 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:27,311 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:27,325 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal.1733770407312, exclude list is [], retry=0 2024-12-09T18:53:27,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:27,331 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:27,331 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:27,360 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal.1733770407312 2024-12-09T18:53:27,360 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:27,360 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 23bfcc9a8cb95c05d9e83db5761e75c0, NAME => 'testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:27,361 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:27,361 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:27,361 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:27,363 INFO [StoreOpener-23bfcc9a8cb95c05d9e83db5761e75c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:27,365 INFO [StoreOpener-23bfcc9a8cb95c05d9e83db5761e75c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23bfcc9a8cb95c05d9e83db5761e75c0 columnFamilyName a 2024-12-09T18:53:27,365 DEBUG [StoreOpener-23bfcc9a8cb95c05d9e83db5761e75c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:27,366 INFO [StoreOpener-23bfcc9a8cb95c05d9e83db5761e75c0-1 {}] regionserver.HStore(327): Store=23bfcc9a8cb95c05d9e83db5761e75c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:27,366 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:27,367 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:27,370 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:27,371 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000002 2024-12-09T18:53:27,404 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:27,409 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000002 2024-12-09T18:53:27,412 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 23bfcc9a8cb95c05d9e83db5761e75c0 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T18:53:27,460 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/.tmp/a/347b788c7c044448910d91956196435b is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733770406986/Put/seqid=0 2024-12-09T18:53:27,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741844_1020 (size=5170) 2024-12-09T18:53:27,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741844_1020 (size=5170) 2024-12-09T18:53:27,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741844_1020 (size=5170) 2024-12-09T18:53:27,472 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/.tmp/a/347b788c7c044448910d91956196435b 2024-12-09T18:53:27,518 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/.tmp/a/347b788c7c044448910d91956196435b as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/a/347b788c7c044448910d91956196435b 2024-12-09T18:53:27,530 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/a/347b788c7c044448910d91956196435b, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T18:53:27,537 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 23bfcc9a8cb95c05d9e83db5761e75c0 in 122ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T18:53:27,539 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/0000000000000000002 2024-12-09T18:53:27,540 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:27,540 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:27,544 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 23bfcc9a8cb95c05d9e83db5761e75c0 2024-12-09T18:53:27,547 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/23bfcc9a8cb95c05d9e83db5761e75c0/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T18:53:27,549 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 23bfcc9a8cb95c05d9e83db5761e75c0; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68485289, jitterRate=0.02051033079624176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T18:53:27,550 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 23bfcc9a8cb95c05d9e83db5761e75c0: Writing region info on filesystem at 1733770407361Initializing all the Stores at 1733770407362 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770407362Obtaining lock to block concurrent updates at 1733770407412 (+50 ms)Preparing flush snapshotting stores in 23bfcc9a8cb95c05d9e83db5761e75c0 at 1733770407412Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733770407415 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733770406864.23bfcc9a8cb95c05d9e83db5761e75c0. at 1733770407415Flushing 23bfcc9a8cb95c05d9e83db5761e75c0/a: creating writer at 1733770407416 (+1 ms)Flushing 23bfcc9a8cb95c05d9e83db5761e75c0/a: appending metadata at 1733770407452 (+36 ms)Flushing 23bfcc9a8cb95c05d9e83db5761e75c0/a: closing flushed file at 1733770407454 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5046dd89: reopening flushed file at 1733770407516 (+62 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 23bfcc9a8cb95c05d9e83db5761e75c0 in 122ms, sequenceid=2, compaction requested=false; wal=null at 1733770407537 (+21 ms)Cleaning up temporary data from old regions at 1733770407540 (+3 ms)Region opened successfully at 1733770407550 (+10 ms) 2024-12-09T18:53:27,586 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=380 (was 369) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:55726 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:55578 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:44144 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:44286 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:51980 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:52116 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:41577/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=214 (was 190) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3504 (was 3584) 2024-12-09T18:53:27,597 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=380, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=214, ProcessCount=11, AvailableMemoryMB=3502 2024-12-09T18:53:27,615 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:27,617 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:27,618 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:27,621 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-32143923, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/hregion-32143923, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:27,635 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-32143923/hregion-32143923.1733770407622, exclude list is [], retry=0 2024-12-09T18:53:27,640 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:27,643 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:27,643 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:27,677 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-32143923/hregion-32143923.1733770407622 2024-12-09T18:53:27,679 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:27,680 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 865ae08f8b5d0281d578a49f62e04386, NAME => 'testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/hbase 2024-12-09T18:53:27,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741846_1022 (size=64) 2024-12-09T18:53:27,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741846_1022 (size=64) 2024-12-09T18:53:27,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741846_1022 (size=64) 2024-12-09T18:53:27,694 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:27,696 INFO [StoreOpener-865ae08f8b5d0281d578a49f62e04386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:27,698 INFO [StoreOpener-865ae08f8b5d0281d578a49f62e04386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 865ae08f8b5d0281d578a49f62e04386 columnFamilyName a 2024-12-09T18:53:27,698 DEBUG [StoreOpener-865ae08f8b5d0281d578a49f62e04386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:27,699 INFO [StoreOpener-865ae08f8b5d0281d578a49f62e04386-1 {}] regionserver.HStore(327): Store=865ae08f8b5d0281d578a49f62e04386/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:27,700 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:27,701 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:27,701 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:27,702 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:27,702 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:27,704 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:27,720 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:27,722 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 865ae08f8b5d0281d578a49f62e04386; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68582718, jitterRate=0.021962136030197144}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T18:53:27,722 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 865ae08f8b5d0281d578a49f62e04386: Writing region info on filesystem at 1733770407694Initializing all the Stores at 1733770407695 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770407695Cleaning up temporary data from old regions at 1733770407702 (+7 ms)Region opened successfully at 1733770407722 (+20 ms) 2024-12-09T18:53:27,723 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 865ae08f8b5d0281d578a49f62e04386, disabling compactions & flushes 2024-12-09T18:53:27,723 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386. 2024-12-09T18:53:27,723 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386. 2024-12-09T18:53:27,723 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386. after waiting 0 ms 2024-12-09T18:53:27,723 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386. 2024-12-09T18:53:27,726 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386. 2024-12-09T18:53:27,726 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 865ae08f8b5d0281d578a49f62e04386: Waiting for close lock at 1733770407723Disabling compacts and flushes for region at 1733770407723Disabling writes for close at 1733770407723Writing region close event to WAL at 1733770407726 (+3 ms)Closed at 1733770407726 2024-12-09T18:53:27,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741845_1021 (size=175) 2024-12-09T18:53:27,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741845_1021 (size=175) 2024-12-09T18:53:27,742 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:27,742 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-32143923:(num 1733770407622) 2024-12-09T18:53:27,743 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T18:53:27,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741845_1021 (size=175) 2024-12-09T18:53:27,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741847_1023 (size=469) 2024-12-09T18:53:27,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741847_1023 (size=469) 2024-12-09T18:53:27,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741847_1023 (size=469) 2024-12-09T18:53:27,802 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T18:53:27,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741848_1024 (size=322) 2024-12-09T18:53:27,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741848_1024 (size=322) 2024-12-09T18:53:27,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741848_1024 (size=322) 2024-12-09T18:53:27,879 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-2, size=322 (322bytes) 2024-12-09T18:53:27,879 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-2 2024-12-09T18:53:27,880 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-2 after 1ms 2024-12-09T18:53:27,908 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:27,908 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-2 took 30ms 2024-12-09T18:53:27,911 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-2 so closing down 2024-12-09T18:53:27,911 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:27,913 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T18:53:27,940 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T18:53:27,941 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:27,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741849_1025 (size=322) 2024-12-09T18:53:27,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741849_1025 (size=322) 2024-12-09T18:53:27,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741849_1025 (size=322) 2024-12-09T18:53:27,952 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T18:53:27,954 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002 2024-12-09T18:53:27,954 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 46 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-09T18:53:27,955 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-2, journal: Splitting hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-2, size=322 (322bytes) at 1733770407879Finishing writing output for hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-2 so closing down at 1733770407911 (+32 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002-wal-2.temp at 1733770407941 (+30 ms)3 split writer threads finished at 1733770407941Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733770407952 (+11 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002 at 1733770407954 (+2 ms)Processed 1 edits across 1 Regions in 46 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733770407954 2024-12-09T18:53:27,968 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-1, size=469 (469bytes) 2024-12-09T18:53:27,968 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-1 2024-12-09T18:53:27,969 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-1 after 1ms 2024-12-09T18:53:27,993 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:27,993 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-1 took 25ms 2024-12-09T18:53:27,996 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-1 so closing down 2024-12-09T18:53:27,996 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:27,998 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T18:53:28,028 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T18:53:28,029 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:28,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741850_1026 (size=469) 2024-12-09T18:53:28,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741850_1026 (size=469) 2024-12-09T18:53:28,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741850_1026 (size=469) 2024-12-09T18:53:28,038 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T18:53:28,065 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:28,067 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002, length=322 2024-12-09T18:53:28,070 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002 2024-12-09T18:53:28,070 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-09T18:53:28,070 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-1, journal: Splitting hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-1, size=469 (469bytes) at 1733770407968Finishing writing output for hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-1 so closing down at 1733770407996 (+28 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000001-wal-1.temp at 1733770408028 (+32 ms)3 split writer threads finished at 1733770408029 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733770408038 (+9 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002 at 1733770408070 (+32 ms)Processed 2 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733770408070 2024-12-09T18:53:28,071 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:28,073 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:28,086 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal.1733770408074, exclude list is [], retry=0 2024-12-09T18:53:28,090 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:28,091 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:28,091 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:28,121 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal.1733770408074 2024-12-09T18:53:28,122 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879)] 2024-12-09T18:53:28,122 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 865ae08f8b5d0281d578a49f62e04386, NAME => 'testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:28,122 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:28,122 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:28,122 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:28,124 INFO [StoreOpener-865ae08f8b5d0281d578a49f62e04386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:28,126 INFO [StoreOpener-865ae08f8b5d0281d578a49f62e04386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 865ae08f8b5d0281d578a49f62e04386 columnFamilyName a 2024-12-09T18:53:28,127 DEBUG [StoreOpener-865ae08f8b5d0281d578a49f62e04386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:28,127 INFO [StoreOpener-865ae08f8b5d0281d578a49f62e04386-1 {}] regionserver.HStore(327): Store=865ae08f8b5d0281d578a49f62e04386/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:28,128 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:28,129 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:28,132 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:28,133 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002 2024-12-09T18:53:28,161 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:28,163 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002 2024-12-09T18:53:28,163 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 865ae08f8b5d0281d578a49f62e04386 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T18:53:28,178 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/.tmp/a/a9c1cee7d6e440a7ad818292efb58e44 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733770407742/Put/seqid=0 2024-12-09T18:53:28,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741852_1028 (size=5170) 2024-12-09T18:53:28,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741852_1028 (size=5170) 2024-12-09T18:53:28,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741852_1028 (size=5170) 2024-12-09T18:53:28,192 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/.tmp/a/a9c1cee7d6e440a7ad818292efb58e44 2024-12-09T18:53:28,203 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/.tmp/a/a9c1cee7d6e440a7ad818292efb58e44 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/a/a9c1cee7d6e440a7ad818292efb58e44 2024-12-09T18:53:28,212 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/a/a9c1cee7d6e440a7ad818292efb58e44, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T18:53:28,212 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 865ae08f8b5d0281d578a49f62e04386 in 49ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T18:53:28,213 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/0000000000000000002 2024-12-09T18:53:28,213 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:28,214 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:28,216 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 865ae08f8b5d0281d578a49f62e04386 2024-12-09T18:53:28,219 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/865ae08f8b5d0281d578a49f62e04386/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T18:53:28,221 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 865ae08f8b5d0281d578a49f62e04386; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62833773, jitterRate=-0.06370382010936737}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T18:53:28,221 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 865ae08f8b5d0281d578a49f62e04386: Writing region info on filesystem at 1733770408122Initializing all the Stores at 1733770408124 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770408124Obtaining lock to block concurrent updates at 1733770408163 (+39 ms)Preparing flush snapshotting stores in 865ae08f8b5d0281d578a49f62e04386 at 1733770408163Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733770408163Flushing stores of testReplayEditsWrittenIntoWAL,,1733770407615.865ae08f8b5d0281d578a49f62e04386. at 1733770408163Flushing 865ae08f8b5d0281d578a49f62e04386/a: creating writer at 1733770408164 (+1 ms)Flushing 865ae08f8b5d0281d578a49f62e04386/a: appending metadata at 1733770408177 (+13 ms)Flushing 865ae08f8b5d0281d578a49f62e04386/a: closing flushed file at 1733770408177Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b6850ba: reopening flushed file at 1733770408201 (+24 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 865ae08f8b5d0281d578a49f62e04386 in 49ms, sequenceid=2, compaction requested=false; wal=null at 1733770408212 (+11 ms)Cleaning up temporary data from old regions at 1733770408214 (+2 ms)Region opened successfully at 1733770408221 (+7 ms) 2024-12-09T18:53:28,237 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=390 (was 380) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:52178 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:55578 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:55808 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:51980 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:44356 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=214 (was 214), ProcessCount=11 (was 11), AvailableMemoryMB=3423 (was 3502) 2024-12-09T18:53:28,249 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=390, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=214, ProcessCount=11, AvailableMemoryMB=3420 2024-12-09T18:53:28,263 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:28,266 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:28,267 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:28,270 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92611662, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/hregion-92611662, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:28,288 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92611662/hregion-92611662.1733770408271, exclude list is [], retry=0 2024-12-09T18:53:28,292 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:28,296 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:28,296 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:28,326 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92611662/hregion-92611662.1733770408271 2024-12-09T18:53:28,327 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:28,327 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9ff425ab9e7eead7ffcbc106fb1dba1e, NAME => 'testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/hbase 2024-12-09T18:53:28,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741854_1030 (size=64) 2024-12-09T18:53:28,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741854_1030 (size=64) 2024-12-09T18:53:28,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741854_1030 (size=64) 2024-12-09T18:53:28,350 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:28,353 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:28,356 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ff425ab9e7eead7ffcbc106fb1dba1e columnFamilyName a 2024-12-09T18:53:28,356 DEBUG [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:28,357 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(327): Store=9ff425ab9e7eead7ffcbc106fb1dba1e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:28,357 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:28,360 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ff425ab9e7eead7ffcbc106fb1dba1e columnFamilyName b 2024-12-09T18:53:28,360 DEBUG [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:28,361 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(327): Store=9ff425ab9e7eead7ffcbc106fb1dba1e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:28,361 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:28,363 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ff425ab9e7eead7ffcbc106fb1dba1e columnFamilyName c 2024-12-09T18:53:28,363 DEBUG [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:28,364 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(327): Store=9ff425ab9e7eead7ffcbc106fb1dba1e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:28,364 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:28,365 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:28,366 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:28,367 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:28,367 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:28,368 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:28,370 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:28,372 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:28,373 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9ff425ab9e7eead7ffcbc106fb1dba1e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68703390, jitterRate=0.023760288953781128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:28,373 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9ff425ab9e7eead7ffcbc106fb1dba1e: Writing region info on filesystem at 1733770408350Initializing all the Stores at 1733770408353 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770408353Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770408353Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770408353Cleaning up temporary data from old regions at 1733770408368 (+15 ms)Region opened successfully at 1733770408373 (+5 ms) 2024-12-09T18:53:28,374 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9ff425ab9e7eead7ffcbc106fb1dba1e, disabling compactions & flushes 2024-12-09T18:53:28,374 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. 2024-12-09T18:53:28,374 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. 2024-12-09T18:53:28,374 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. after waiting 0 ms 2024-12-09T18:53:28,374 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. 2024-12-09T18:53:28,374 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. 2024-12-09T18:53:28,374 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9ff425ab9e7eead7ffcbc106fb1dba1e: Waiting for close lock at 1733770408373Disabling compacts and flushes for region at 1733770408373Disabling writes for close at 1733770408374 (+1 ms)Writing region close event to WAL at 1733770408374Closed at 1733770408374 2024-12-09T18:53:28,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741853_1029 (size=175) 2024-12-09T18:53:28,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741853_1029 (size=175) 2024-12-09T18:53:28,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741853_1029 (size=175) 2024-12-09T18:53:28,383 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:28,383 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92611662:(num 1733770408271) 2024-12-09T18:53:28,383 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:28,386 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:28,398 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386, exclude list is [], retry=0 2024-12-09T18:53:28,401 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:28,402 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:28,402 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:28,438 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 2024-12-09T18:53:28,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:28,639 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386, size=0 (0bytes) 2024-12-09T18:53:28,639 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 might be still open, length is 0 2024-12-09T18:53:28,639 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 2024-12-09T18:53:28,640 WARN [IPC Server handler 0 on default port 41577 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-09T18:53:28,641 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 after 2ms 2024-12-09T18:53:31,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:52210 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37407:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52210 dst: /127.0.0.1:37407 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37407 remote=/127.0.0.1:52210]. Total timeout mills is 60000, 57300 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:31,309 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:44382 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38029:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44382 dst: /127.0.0.1:38029 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:31,309 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:55824 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36059:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55824 dst: /127.0.0.1:36059 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:31,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741855_1032 (size=497775) 2024-12-09T18:53:31,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741855_1032 (size=497775) 2024-12-09T18:53:31,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741855_1032 (size=497775) 2024-12-09T18:53:31,914 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T18:53:31,966 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T18:53:32,643 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 after 4003ms 2024-12-09T18:53:32,670 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:32,671 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 took 4033ms 2024-12-09T18:53:32,676 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733770408386.temp 2024-12-09T18:53:32,702 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000000001-wal.1733770408386.temp 2024-12-09T18:53:32,761 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386; continuing. 2024-12-09T18:53:32,761 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 so closing down 2024-12-09T18:53:32,761 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:32,761 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:32,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741856_1033 (size=497783) 2024-12-09T18:53:32,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741856_1033 (size=497783) 2024-12-09T18:53:32,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741856_1033 (size=497783) 2024-12-09T18:53:32,765 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000000001-wal.1733770408386.temp (wrote 3002 edits, skipped 0 edits in 41 ms) 2024-12-09T18:53:32,767 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000000001-wal.1733770408386.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000003002 2024-12-09T18:53:32,767 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 95 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T18:53:32,767 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386, journal: Splitting hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386, size=0 (0bytes) at 1733770408639Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000000001-wal.1733770408386.temp at 1733770412702 (+4063 ms)Split 1024 edits, skipped 0 edits. at 1733770412717 (+15 ms)Split 2048 edits, skipped 0 edits. at 1733770412742 (+25 ms)Finishing writing output for hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 so closing down at 1733770412761 (+19 ms)3 split writer threads finished at 1733770412761Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000000001-wal.1733770408386.temp (wrote 3002 edits, skipped 0 edits in 41 ms) at 1733770412765 (+4 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000000001-wal.1733770408386.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000003002 at 1733770412767 (+2 ms)Processed 3002 edits across 1 Regions in 95 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386, size=0, length=0, corrupted=false, cancelled=false at 1733770412767 2024-12-09T18:53:32,769 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 to hdfs://localhost:41577/hbase/oldWALs/wal.1733770408386 2024-12-09T18:53:32,771 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000003002 2024-12-09T18:53:32,771 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:32,773 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:32,786 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770412774, exclude list is [], retry=0 2024-12-09T18:53:32,789 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:32,789 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:32,789 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:32,813 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770412774 2024-12-09T18:53:32,813 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071)] 2024-12-09T18:53:32,814 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:32,816 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:32,817 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ff425ab9e7eead7ffcbc106fb1dba1e columnFamilyName a 2024-12-09T18:53:32,817 DEBUG [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:32,818 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(327): Store=9ff425ab9e7eead7ffcbc106fb1dba1e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:32,818 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:32,819 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ff425ab9e7eead7ffcbc106fb1dba1e columnFamilyName b 2024-12-09T18:53:32,819 DEBUG [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:32,820 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(327): Store=9ff425ab9e7eead7ffcbc106fb1dba1e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:32,820 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:32,821 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ff425ab9e7eead7ffcbc106fb1dba1e columnFamilyName c 2024-12-09T18:53:32,821 DEBUG [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:32,822 INFO [StoreOpener-9ff425ab9e7eead7ffcbc106fb1dba1e-1 {}] regionserver.HStore(327): Store=9ff425ab9e7eead7ffcbc106fb1dba1e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:32,822 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:32,823 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:32,825 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:32,825 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000003002 2024-12-09T18:53:32,852 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:32,895 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T18:53:33,213 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9ff425ab9e7eead7ffcbc106fb1dba1e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T18:53:33,244 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/a/52ab9342fb324a0685d8149cb768559b is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733770408448/Put/seqid=0 2024-12-09T18:53:33,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741858_1035 (size=50463) 2024-12-09T18:53:33,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741858_1035 (size=50463) 2024-12-09T18:53:33,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741858_1035 (size=50463) 2024-12-09T18:53:33,254 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/a/52ab9342fb324a0685d8149cb768559b 2024-12-09T18:53:33,262 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/a/52ab9342fb324a0685d8149cb768559b as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/a/52ab9342fb324a0685d8149cb768559b 2024-12-09T18:53:33,268 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/a/52ab9342fb324a0685d8149cb768559b, entries=754, sequenceid=754, filesize=49.3 K 2024-12-09T18:53:33,269 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 9ff425ab9e7eead7ffcbc106fb1dba1e in 55ms, sequenceid=754, compaction requested=false; wal=null 2024-12-09T18:53:33,284 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T18:53:33,284 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9ff425ab9e7eead7ffcbc106fb1dba1e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T18:53:33,291 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/a/fa49ac79517d406283d7ebfcf95e2540 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733770408498/Put/seqid=0 2024-12-09T18:53:33,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741859_1036 (size=20072) 2024-12-09T18:53:33,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741859_1036 (size=20072) 2024-12-09T18:53:33,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741859_1036 (size=20072) 2024-12-09T18:53:33,300 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/a/fa49ac79517d406283d7ebfcf95e2540 2024-12-09T18:53:33,323 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/b/fcdf7fc3d8c642698057f5beef4eb703 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733770408528/Put/seqid=0 2024-12-09T18:53:33,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741860_1037 (size=35835) 2024-12-09T18:53:33,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741860_1037 (size=35835) 2024-12-09T18:53:33,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741860_1037 (size=35835) 2024-12-09T18:53:33,330 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/b/fcdf7fc3d8c642698057f5beef4eb703 2024-12-09T18:53:33,338 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/a/fa49ac79517d406283d7ebfcf95e2540 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/a/fa49ac79517d406283d7ebfcf95e2540 2024-12-09T18:53:33,346 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/a/fa49ac79517d406283d7ebfcf95e2540, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-09T18:53:33,348 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/b/fcdf7fc3d8c642698057f5beef4eb703 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/b/fcdf7fc3d8c642698057f5beef4eb703 2024-12-09T18:53:33,356 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/b/fcdf7fc3d8c642698057f5beef4eb703, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-09T18:53:33,356 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 9ff425ab9e7eead7ffcbc106fb1dba1e in 72ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-09T18:53:33,376 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T18:53:33,377 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9ff425ab9e7eead7ffcbc106fb1dba1e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T18:53:33,385 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/b/512183cd728c4bcea103b2fb406e80d2 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733770408546/Put/seqid=0 2024-12-09T18:53:33,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741861_1038 (size=35082) 2024-12-09T18:53:33,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741861_1038 (size=35082) 2024-12-09T18:53:33,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741861_1038 (size=35082) 2024-12-09T18:53:33,397 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/b/512183cd728c4bcea103b2fb406e80d2 2024-12-09T18:53:33,429 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/c/54df556dc70f4b3c82cb8221f8856ed7 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733770408568/Put/seqid=0 2024-12-09T18:53:33,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741862_1039 (size=20825) 2024-12-09T18:53:33,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741862_1039 (size=20825) 2024-12-09T18:53:33,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741862_1039 (size=20825) 2024-12-09T18:53:33,441 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/c/54df556dc70f4b3c82cb8221f8856ed7 2024-12-09T18:53:33,450 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/b/512183cd728c4bcea103b2fb406e80d2 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/b/512183cd728c4bcea103b2fb406e80d2 2024-12-09T18:53:33,458 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/b/512183cd728c4bcea103b2fb406e80d2, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-09T18:53:33,460 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/c/54df556dc70f4b3c82cb8221f8856ed7 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/c/54df556dc70f4b3c82cb8221f8856ed7 2024-12-09T18:53:33,468 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/c/54df556dc70f4b3c82cb8221f8856ed7, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-09T18:53:33,468 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 9ff425ab9e7eead7ffcbc106fb1dba1e in 91ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-09T18:53:33,478 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733770408606/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. 2024-12-09T18:53:33,482 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000003002 2024-12-09T18:53:33,482 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T18:53:33,483 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9ff425ab9e7eead7ffcbc106fb1dba1e 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-09T18:53:33,490 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/c/ce96f53cb9274e6c9ebefeff1776da1b is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733770408574/Put/seqid=0 2024-12-09T18:53:33,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741863_1040 (size=50301) 2024-12-09T18:53:33,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741863_1040 (size=50301) 2024-12-09T18:53:33,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741863_1040 (size=50301) 2024-12-09T18:53:33,503 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/c/ce96f53cb9274e6c9ebefeff1776da1b 2024-12-09T18:53:33,510 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ce96f53cb9274e6c9ebefeff1776da1b 2024-12-09T18:53:33,512 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/.tmp/c/ce96f53cb9274e6c9ebefeff1776da1b as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/c/ce96f53cb9274e6c9ebefeff1776da1b 2024-12-09T18:53:33,519 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ce96f53cb9274e6c9ebefeff1776da1b 2024-12-09T18:53:33,519 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/c/ce96f53cb9274e6c9ebefeff1776da1b, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-09T18:53:33,520 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 9ff425ab9e7eead7ffcbc106fb1dba1e in 37ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-09T18:53:33,521 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/0000000000000003002 2024-12-09T18:53:33,522 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:33,522 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:33,523 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T18:53:33,526 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9ff425ab9e7eead7ffcbc106fb1dba1e 2024-12-09T18:53:33,529 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenIntoWAL/9ff425ab9e7eead7ffcbc106fb1dba1e/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-09T18:53:33,530 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T18:53:33,530 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9ff425ab9e7eead7ffcbc106fb1dba1e; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74076640, jitterRate=0.10382795333862305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T18:53:33,530 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9ff425ab9e7eead7ffcbc106fb1dba1e: Writing region info on filesystem at 1733770412814Initializing all the Stores at 1733770412815 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770412815Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770412815Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770412815Cleaning up temporary data from old regions at 1733770413522 (+707 ms)Region opened successfully at 1733770413530 (+8 ms) 2024-12-09T18:53:33,531 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T18:53:33,533 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T18:53:33,533 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T18:53:33,533 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T18:53:33,534 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T18:53:33,535 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-09T18:53:33,535 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-09T18:53:33,590 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9ff425ab9e7eead7ffcbc106fb1dba1e, disabling compactions & flushes 2024-12-09T18:53:33,590 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. 2024-12-09T18:53:33,590 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. 2024-12-09T18:53:33,590 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. after waiting 0 ms 2024-12-09T18:53:33,590 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. 2024-12-09T18:53:33,592 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733770408264.9ff425ab9e7eead7ffcbc106fb1dba1e. 2024-12-09T18:53:33,593 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9ff425ab9e7eead7ffcbc106fb1dba1e: Waiting for close lock at 1733770413590Disabling compacts and flushes for region at 1733770413590Disabling writes for close at 1733770413590Writing region close event to WAL at 1733770413592 (+2 ms)Closed at 1733770413592 2024-12-09T18:53:33,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741857_1034 (size=175) 2024-12-09T18:53:33,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741857_1034 (size=175) 2024-12-09T18:53:33,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741857_1034 (size=175) 2024-12-09T18:53:33,599 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:33,599 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733770412774) 2024-12-09T18:53:33,612 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=404 (was 390) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34513 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_393879971_22 at /127.0.0.1:49520 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_393879971_22 at /127.0.0.1:56470 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:41577 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:34513 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33619 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_393879971_22 at /127.0.0.1:56448 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:41577 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:33619 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_393879971_22 at /127.0.0.1:32890 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=197 (was 214), ProcessCount=11 (was 11), AvailableMemoryMB=3183 (was 3420) 2024-12-09T18:53:33,625 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=3183 2024-12-09T18:53:33,646 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:33,649 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:33,650 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:33,654 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79108042, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/hregion-79108042, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:33,672 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79108042/hregion-79108042.1733770413655, exclude list is [], retry=0 2024-12-09T18:53:33,675 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:33,676 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:33,676 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:33,702 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79108042/hregion-79108042.1733770413655 2024-12-09T18:53:33,703 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071)] 2024-12-09T18:53:33,703 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d2aa89a395bd2cc162c8552d3e85a36e, NAME => 'test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/hbase 2024-12-09T18:53:33,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741865_1042 (size=43) 2024-12-09T18:53:33,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741865_1042 (size=43) 2024-12-09T18:53:33,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741865_1042 (size=43) 2024-12-09T18:53:33,718 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:33,719 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:33,721 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2aa89a395bd2cc162c8552d3e85a36e columnFamilyName a 2024-12-09T18:53:33,721 DEBUG [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:33,721 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(327): Store=d2aa89a395bd2cc162c8552d3e85a36e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:33,721 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:33,723 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2aa89a395bd2cc162c8552d3e85a36e columnFamilyName b 2024-12-09T18:53:33,723 DEBUG [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:33,724 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(327): Store=d2aa89a395bd2cc162c8552d3e85a36e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:33,724 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:33,726 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2aa89a395bd2cc162c8552d3e85a36e columnFamilyName c 2024-12-09T18:53:33,726 DEBUG [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:33,726 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(327): Store=d2aa89a395bd2cc162c8552d3e85a36e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:33,727 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:33,727 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:33,728 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:33,729 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:33,730 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:33,730 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:33,732 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:33,734 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:33,735 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d2aa89a395bd2cc162c8552d3e85a36e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69993124, jitterRate=0.04297882318496704}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:33,736 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d2aa89a395bd2cc162c8552d3e85a36e: Writing region info on filesystem at 1733770413718Initializing all the Stores at 1733770413719 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770413719Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770413719Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770413719Cleaning up temporary data from old regions at 1733770413730 (+11 ms)Region opened successfully at 1733770413736 (+6 ms) 2024-12-09T18:53:33,736 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d2aa89a395bd2cc162c8552d3e85a36e, disabling compactions & flushes 2024-12-09T18:53:33,736 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. 2024-12-09T18:53:33,736 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. 2024-12-09T18:53:33,736 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. after waiting 0 ms 2024-12-09T18:53:33,736 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. 2024-12-09T18:53:33,737 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. 2024-12-09T18:53:33,737 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d2aa89a395bd2cc162c8552d3e85a36e: Waiting for close lock at 1733770413736Disabling compacts and flushes for region at 1733770413736Disabling writes for close at 1733770413736Writing region close event to WAL at 1733770413737 (+1 ms)Closed at 1733770413737 2024-12-09T18:53:33,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741864_1041 (size=175) 2024-12-09T18:53:33,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741864_1041 (size=175) 2024-12-09T18:53:33,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741864_1041 (size=175) 2024-12-09T18:53:33,745 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:33,745 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79108042:(num 1733770413655) 2024-12-09T18:53:33,745 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:33,748 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:33,761 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748, exclude list is [], retry=0 2024-12-09T18:53:33,763 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:33,764 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:33,764 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:33,787 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748 2024-12-09T18:53:33,788 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:33,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741866_1043 (size=371388) 2024-12-09T18:53:33,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741866_1043 (size=371388) 2024-12-09T18:53:33,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741866_1043 (size=371388) 2024-12-09T18:53:33,945 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748, size=362.7 K (371388bytes) 2024-12-09T18:53:33,945 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748 2024-12-09T18:53:33,946 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748 after 0ms 2024-12-09T18:53:33,976 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:33,978 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748 took 33ms 2024-12-09T18:53:33,982 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733770413748.temp 2024-12-09T18:53:34,017 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748 so closing down 2024-12-09T18:53:34,017 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:34,023 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000000001-wal.1733770413748.temp 2024-12-09T18:53:34,043 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:34,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741867_1044 (size=371388) 2024-12-09T18:53:34,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741867_1044 (size=371388) 2024-12-09T18:53:34,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741867_1044 (size=371388) 2024-12-09T18:53:34,047 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000000001-wal.1733770413748.temp (wrote 3000 edits, skipped 0 edits in 19 ms) 2024-12-09T18:53:34,049 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000000001-wal.1733770413748.temp to hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003000 2024-12-09T18:53:34,049 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-09T18:53:34,049 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748, journal: Splitting hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748, size=362.7 K (371388bytes) at 1733770413945Split 1024 edits, skipped 0 edits. at 1733770413992 (+47 ms)Split 2048 edits, skipped 0 edits. at 1733770414004 (+12 ms)Finishing writing output for hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748 so closing down at 1733770414017 (+13 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000000001-wal.1733770413748.temp at 1733770414023 (+6 ms)3 split writer threads finished at 1733770414043 (+20 ms)Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000000001-wal.1733770413748.temp (wrote 3000 edits, skipped 0 edits in 19 ms) at 1733770414047 (+4 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000000001-wal.1733770413748.temp to hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003000 at 1733770414049 (+2 ms)Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733770414049 2024-12-09T18:53:34,051 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770413748 to hdfs://localhost:41577/hbase/oldWALs/wal.1733770413748 2024-12-09T18:53:34,052 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003000 2024-12-09T18:53:34,052 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:34,055 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:34,074 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055, exclude list is [], retry=0 2024-12-09T18:53:34,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:34,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:34,079 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:34,119 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055 2024-12-09T18:53:34,119 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071)] 2024-12-09T18:53:34,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741868_1045 (size=371515) 2024-12-09T18:53:34,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741868_1045 (size=371515) 2024-12-09T18:53:34,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741868_1045 (size=371515) 2024-12-09T18:53:34,278 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055, size=362.8 K (371515bytes) 2024-12-09T18:53:34,278 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055 2024-12-09T18:53:34,279 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055 after 1ms 2024-12-09T18:53:34,308 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:34,310 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055 took 32ms 2024-12-09T18:53:34,314 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733770414055.temp 2024-12-09T18:53:34,371 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055 so closing down 2024-12-09T18:53:34,371 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:34,372 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003001-wal.1733770414055.temp 2024-12-09T18:53:34,388 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:34,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741869_1046 (size=371515) 2024-12-09T18:53:34,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741869_1046 (size=371515) 2024-12-09T18:53:34,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741869_1046 (size=371515) 2024-12-09T18:53:34,392 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003001-wal.1733770414055.temp (wrote 3000 edits, skipped 0 edits in 15 ms) 2024-12-09T18:53:34,394 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003001-wal.1733770414055.temp to hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000006000 2024-12-09T18:53:34,394 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 84 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-09T18:53:34,394 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055, journal: Splitting hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055, size=362.8 K (371515bytes) at 1733770414278Split 1024 edits, skipped 0 edits. at 1733770414328 (+50 ms)Split 2048 edits, skipped 0 edits. at 1733770414360 (+32 ms)Finishing writing output for hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055 so closing down at 1733770414371 (+11 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003001-wal.1733770414055.temp at 1733770414372 (+1 ms)3 split writer threads finished at 1733770414388 (+16 ms)Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003001-wal.1733770414055.temp (wrote 3000 edits, skipped 0 edits in 15 ms) at 1733770414392 (+4 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003001-wal.1733770414055.temp to hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000006000 at 1733770414394 (+2 ms)Processed 3000 edits across 1 Regions in 84 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733770414394 2024-12-09T18:53:34,396 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414055 to hdfs://localhost:41577/hbase/oldWALs/wal.1733770414055 2024-12-09T18:53:34,398 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000006000 2024-12-09T18:53:34,398 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:34,400 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/test2727-manual,16010,1733770413645, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:34,418 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414401, exclude list is [], retry=0 2024-12-09T18:53:34,421 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:34,421 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:34,422 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:34,458 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733770413645/wal.1733770414401 2024-12-09T18:53:34,459 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:34,459 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d2aa89a395bd2cc162c8552d3e85a36e, NAME => 'test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:34,459 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:34,459 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,459 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,461 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,462 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2aa89a395bd2cc162c8552d3e85a36e columnFamilyName a 2024-12-09T18:53:34,462 DEBUG [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:34,463 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(327): Store=d2aa89a395bd2cc162c8552d3e85a36e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:34,463 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,464 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2aa89a395bd2cc162c8552d3e85a36e columnFamilyName b 2024-12-09T18:53:34,465 DEBUG [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:34,465 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(327): Store=d2aa89a395bd2cc162c8552d3e85a36e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:34,465 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,467 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2aa89a395bd2cc162c8552d3e85a36e columnFamilyName c 2024-12-09T18:53:34,467 DEBUG [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:34,467 INFO [StoreOpener-d2aa89a395bd2cc162c8552d3e85a36e-1 {}] regionserver.HStore(327): Store=d2aa89a395bd2cc162c8552d3e85a36e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:34,468 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,469 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,472 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003000 2024-12-09T18:53:34,498 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:34,526 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003000 2024-12-09T18:53:34,527 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000006000 2024-12-09T18:53:34,566 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:34,608 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000006000 2024-12-09T18:53:34,609 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d2aa89a395bd2cc162c8552d3e85a36e 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-09T18:53:34,629 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/.tmp/a/7e4c77f3f83e4c9d82c7be4809cbfd19 is 41, key is test2727/a:100/1733770414124/Put/seqid=0 2024-12-09T18:53:34,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741871_1048 (size=84227) 2024-12-09T18:53:34,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741871_1048 (size=84227) 2024-12-09T18:53:34,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741871_1048 (size=84227) 2024-12-09T18:53:34,636 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/.tmp/a/7e4c77f3f83e4c9d82c7be4809cbfd19 2024-12-09T18:53:34,669 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/.tmp/b/dfea72f835a9425fa891c12ea2c8b973 is 41, key is test2727/b:100/1733770414174/Put/seqid=0 2024-12-09T18:53:34,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741872_1049 (size=84609) 2024-12-09T18:53:34,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741872_1049 (size=84609) 2024-12-09T18:53:34,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741872_1049 (size=84609) 2024-12-09T18:53:34,678 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/.tmp/b/dfea72f835a9425fa891c12ea2c8b973 2024-12-09T18:53:34,707 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/.tmp/c/61d7080f7b6c47f48f9031bd52e81e87 is 41, key is test2727/c:100/1733770414222/Put/seqid=0 2024-12-09T18:53:34,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741873_1050 (size=84609) 2024-12-09T18:53:34,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741873_1050 (size=84609) 2024-12-09T18:53:34,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741873_1050 (size=84609) 2024-12-09T18:53:34,713 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/.tmp/c/61d7080f7b6c47f48f9031bd52e81e87 2024-12-09T18:53:34,722 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/.tmp/a/7e4c77f3f83e4c9d82c7be4809cbfd19 as hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/a/7e4c77f3f83e4c9d82c7be4809cbfd19 2024-12-09T18:53:34,730 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/a/7e4c77f3f83e4c9d82c7be4809cbfd19, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-09T18:53:34,732 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/.tmp/b/dfea72f835a9425fa891c12ea2c8b973 as hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/b/dfea72f835a9425fa891c12ea2c8b973 2024-12-09T18:53:34,740 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/b/dfea72f835a9425fa891c12ea2c8b973, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T18:53:34,742 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/.tmp/c/61d7080f7b6c47f48f9031bd52e81e87 as hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/c/61d7080f7b6c47f48f9031bd52e81e87 2024-12-09T18:53:34,749 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/c/61d7080f7b6c47f48f9031bd52e81e87, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T18:53:34,750 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for d2aa89a395bd2cc162c8552d3e85a36e in 141ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-09T18:53:34,751 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000003000 2024-12-09T18:53:34,751 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/0000000000000006000 2024-12-09T18:53:34,753 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,753 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,754 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:34,756 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d2aa89a395bd2cc162c8552d3e85a36e 2024-12-09T18:53:34,759 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/test2727/d2aa89a395bd2cc162c8552d3e85a36e/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-09T18:53:34,761 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d2aa89a395bd2cc162c8552d3e85a36e; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71535336, jitterRate=0.06595957279205322}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:34,762 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d2aa89a395bd2cc162c8552d3e85a36e: Writing region info on filesystem at 1733770414459Initializing all the Stores at 1733770414461 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770414461Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770414461Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770414461Obtaining lock to block concurrent updates at 1733770414609 (+148 ms)Preparing flush snapshotting stores in d2aa89a395bd2cc162c8552d3e85a36e at 1733770414609Finished memstore snapshotting test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733770414609Flushing stores of test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. at 1733770414609Flushing d2aa89a395bd2cc162c8552d3e85a36e/a: creating writer at 1733770414609Flushing d2aa89a395bd2cc162c8552d3e85a36e/a: appending metadata at 1733770414628 (+19 ms)Flushing d2aa89a395bd2cc162c8552d3e85a36e/a: closing flushed file at 1733770414628Flushing d2aa89a395bd2cc162c8552d3e85a36e/b: creating writer at 1733770414643 (+15 ms)Flushing d2aa89a395bd2cc162c8552d3e85a36e/b: appending metadata at 1733770414667 (+24 ms)Flushing d2aa89a395bd2cc162c8552d3e85a36e/b: closing flushed file at 1733770414667Flushing d2aa89a395bd2cc162c8552d3e85a36e/c: creating writer at 1733770414684 (+17 ms)Flushing d2aa89a395bd2cc162c8552d3e85a36e/c: appending metadata at 1733770414706 (+22 ms)Flushing d2aa89a395bd2cc162c8552d3e85a36e/c: closing flushed file at 1733770414706Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ecd2d35: reopening flushed file at 1733770414720 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d17db48: reopening flushed file at 1733770414730 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4462d2d1: reopening flushed file at 1733770414740 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for d2aa89a395bd2cc162c8552d3e85a36e in 141ms, sequenceid=6000, compaction requested=false; wal=null at 1733770414750 (+10 ms)Cleaning up temporary data from old regions at 1733770414753 (+3 ms)Region opened successfully at 1733770414762 (+9 ms) 2024-12-09T18:53:34,764 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-09T18:53:34,764 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d2aa89a395bd2cc162c8552d3e85a36e, disabling compactions & flushes 2024-12-09T18:53:34,764 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. 2024-12-09T18:53:34,764 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. 2024-12-09T18:53:34,764 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. after waiting 0 ms 2024-12-09T18:53:34,764 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. 2024-12-09T18:53:34,766 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733770413647.d2aa89a395bd2cc162c8552d3e85a36e. 2024-12-09T18:53:34,766 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d2aa89a395bd2cc162c8552d3e85a36e: Waiting for close lock at 1733770414764Disabling compacts and flushes for region at 1733770414764Disabling writes for close at 1733770414764Writing region close event to WAL at 1733770414766 (+2 ms)Closed at 1733770414766 2024-12-09T18:53:34,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741870_1047 (size=175) 2024-12-09T18:53:34,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741870_1047 (size=175) 2024-12-09T18:53:34,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741870_1047 (size=175) 2024-12-09T18:53:34,772 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:34,772 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733770414401) 2024-12-09T18:53:34,786 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=408 (was 404) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:33038 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:56448 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=197 (was 197), ProcessCount=11 (was 11), AvailableMemoryMB=2949 (was 3183) 2024-12-09T18:53:34,800 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=408, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=2949 2024-12-09T18:53:34,817 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:34,824 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:34,824 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733770414824 2024-12-09T18:53:34,851 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824 2024-12-09T18:53:34,853 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:34,854 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 88f3f629424263defefafef2168b2635, NAME => 'testSequentialEditLogSeqNum,,1733770414818.88f3f629424263defefafef2168b2635.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:34,854 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733770414818.88f3f629424263defefafef2168b2635.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:34,854 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 88f3f629424263defefafef2168b2635 2024-12-09T18:53:34,854 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 88f3f629424263defefafef2168b2635 2024-12-09T18:53:34,855 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635 doesn't exist for region: 88f3f629424263defefafef2168b2635 on table testSequentialEditLogSeqNum 2024-12-09T18:53:34,855 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 88f3f629424263defefafef2168b2635 on table testSequentialEditLogSeqNum 2024-12-09T18:53:34,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741875_1052 (size=62) 2024-12-09T18:53:34,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741875_1052 (size=62) 2024-12-09T18:53:34,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741875_1052 (size=62) 2024-12-09T18:53:34,867 INFO [StoreOpener-88f3f629424263defefafef2168b2635-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 88f3f629424263defefafef2168b2635 2024-12-09T18:53:34,868 INFO [StoreOpener-88f3f629424263defefafef2168b2635-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88f3f629424263defefafef2168b2635 columnFamilyName a 2024-12-09T18:53:34,868 DEBUG [StoreOpener-88f3f629424263defefafef2168b2635-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:34,869 INFO [StoreOpener-88f3f629424263defefafef2168b2635-1 {}] regionserver.HStore(327): Store=88f3f629424263defefafef2168b2635/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:34,869 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 88f3f629424263defefafef2168b2635 2024-12-09T18:53:34,869 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635 2024-12-09T18:53:34,870 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635 2024-12-09T18:53:34,870 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 88f3f629424263defefafef2168b2635 2024-12-09T18:53:34,870 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 88f3f629424263defefafef2168b2635 2024-12-09T18:53:34,872 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 88f3f629424263defefafef2168b2635 2024-12-09T18:53:34,874 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:34,875 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 88f3f629424263defefafef2168b2635; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73967371, jitterRate=0.10219971835613251}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T18:53:34,875 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 88f3f629424263defefafef2168b2635: Writing region info on filesystem at 1733770414854Initializing all the Stores at 1733770414866 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770414866Cleaning up temporary data from old regions at 1733770414870 (+4 ms)Region opened successfully at 1733770414875 (+5 ms) 2024-12-09T18:53:34,886 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 88f3f629424263defefafef2168b2635 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-09T18:53:34,907 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/.tmp/a/8734507f25cf4ecb9f9221b3dbd9bd4f is 81, key is testSequentialEditLogSeqNum/a:x0/1733770414875/Put/seqid=0 2024-12-09T18:53:34,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741876_1053 (size=5833) 2024-12-09T18:53:34,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741876_1053 (size=5833) 2024-12-09T18:53:34,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741876_1053 (size=5833) 2024-12-09T18:53:34,915 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/.tmp/a/8734507f25cf4ecb9f9221b3dbd9bd4f 2024-12-09T18:53:34,922 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/.tmp/a/8734507f25cf4ecb9f9221b3dbd9bd4f as hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/a/8734507f25cf4ecb9f9221b3dbd9bd4f 2024-12-09T18:53:34,929 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/a/8734507f25cf4ecb9f9221b3dbd9bd4f, entries=10, sequenceid=13, filesize=5.7 K 2024-12-09T18:53:34,931 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 88f3f629424263defefafef2168b2635 in 45ms, sequenceid=13, compaction requested=false 2024-12-09T18:53:34,931 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 88f3f629424263defefafef2168b2635: 2024-12-09T18:53:34,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T18:53:34,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T18:53:34,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T18:53:34,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T18:53:34,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T18:53:34,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741874_1051 (size=3256) 2024-12-09T18:53:34,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741874_1051 (size=3256) 2024-12-09T18:53:34,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741874_1051 (size=3256) 2024-12-09T18:53:34,963 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824, size=3.2 K (3256bytes) 2024-12-09T18:53:34,964 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824 2024-12-09T18:53:34,964 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824 after 0ms 2024-12-09T18:53:35,005 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:35,006 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824 took 42ms 2024-12-09T18:53:35,009 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824 so closing down 2024-12-09T18:53:35,009 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:35,010 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733770414824.temp 2024-12-09T18:53:35,049 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/recovered.edits/0000000000000000003-wal.1733770414824.temp 2024-12-09T18:53:35,050 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:35,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741877_1054 (size=2650) 2024-12-09T18:53:35,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741877_1054 (size=2650) 2024-12-09T18:53:35,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741877_1054 (size=2650) 2024-12-09T18:53:35,058 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/recovered.edits/0000000000000000003-wal.1733770414824.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-09T18:53:35,059 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/recovered.edits/0000000000000000003-wal.1733770414824.temp to hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/recovered.edits/0000000000000000020 2024-12-09T18:53:35,060 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 53 ms; skipped=2; WAL=hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-09T18:53:35,060 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824, journal: Splitting hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824, size=3.2 K (3256bytes) at 1733770414963Finishing writing output for hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824 so closing down at 1733770415009 (+46 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/recovered.edits/0000000000000000003-wal.1733770414824.temp at 1733770415049 (+40 ms)3 split writer threads finished at 1733770415050 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/recovered.edits/0000000000000000003-wal.1733770414824.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733770415058 (+8 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/recovered.edits/0000000000000000003-wal.1733770414824.temp to hdfs://localhost:41577/hbase/data/default/testSequentialEditLogSeqNum/88f3f629424263defefafef2168b2635/recovered.edits/0000000000000000020 at 1733770415059 (+1 ms)Processed 17 edits across 1 Regions in 53 ms; skipped=2; WAL=hdfs://localhost:41577/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733770414817/wal.1733770414824, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733770415060 (+1 ms) 2024-12-09T18:53:35,074 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=413 (was 408) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:49520 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:33038 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:56448 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=197 (was 197), ProcessCount=11 (was 11), AvailableMemoryMB=2942 (was 2949) 2024-12-09T18:53:35,088 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=2937 2024-12-09T18:53:35,108 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:35,111 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:35,144 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:35,147 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-75323421, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/hregion-75323421, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:35,165 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-75323421/hregion-75323421.1733770415148, exclude list is [], retry=0 2024-12-09T18:53:35,168 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:35,168 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:35,169 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:35,206 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-75323421/hregion-75323421.1733770415148 2024-12-09T18:53:35,206 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:35,207 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f80d7a9a346d7ab3b086bf0cb6c5225b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/hbase 2024-12-09T18:53:35,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741879_1056 (size=70) 2024-12-09T18:53:35,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741879_1056 (size=70) 2024-12-09T18:53:35,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741879_1056 (size=70) 2024-12-09T18:53:35,220 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:35,222 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,224 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f80d7a9a346d7ab3b086bf0cb6c5225b columnFamilyName a 2024-12-09T18:53:35,224 DEBUG [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:35,225 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(327): Store=f80d7a9a346d7ab3b086bf0cb6c5225b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:35,225 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,226 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f80d7a9a346d7ab3b086bf0cb6c5225b columnFamilyName b 2024-12-09T18:53:35,226 DEBUG [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:35,227 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(327): Store=f80d7a9a346d7ab3b086bf0cb6c5225b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:35,228 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,229 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f80d7a9a346d7ab3b086bf0cb6c5225b columnFamilyName c 2024-12-09T18:53:35,229 DEBUG [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:35,230 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(327): Store=f80d7a9a346d7ab3b086bf0cb6c5225b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:35,230 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,232 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,232 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,233 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:35,234 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,236 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:35,237 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f80d7a9a346d7ab3b086bf0cb6c5225b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74300415, jitterRate=0.1071624606847763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:35,238 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f80d7a9a346d7ab3b086bf0cb6c5225b: Writing region info on filesystem at 1733770415220Initializing all the Stores at 1733770415221 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770415221Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770415221Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770415222 (+1 ms)Cleaning up temporary data from old regions at 1733770415232 (+10 ms)Region opened successfully at 1733770415237 (+5 ms) 2024-12-09T18:53:35,238 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f80d7a9a346d7ab3b086bf0cb6c5225b, disabling compactions & flushes 2024-12-09T18:53:35,238 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. 2024-12-09T18:53:35,238 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. 2024-12-09T18:53:35,238 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. after waiting 0 ms 2024-12-09T18:53:35,238 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. 2024-12-09T18:53:35,238 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. 2024-12-09T18:53:35,238 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f80d7a9a346d7ab3b086bf0cb6c5225b: Waiting for close lock at 1733770415238Disabling compacts and flushes for region at 1733770415238Disabling writes for close at 1733770415238Writing region close event to WAL at 1733770415238Closed at 1733770415238 2024-12-09T18:53:35,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741878_1055 (size=175) 2024-12-09T18:53:35,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741878_1055 (size=175) 2024-12-09T18:53:35,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741878_1055 (size=175) 2024-12-09T18:53:35,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:35,244 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-75323421:(num 1733770415148) 2024-12-09T18:53:35,244 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:35,246 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:35,259 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247, exclude list is [], retry=0 2024-12-09T18:53:35,262 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:35,262 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:35,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:35,288 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 2024-12-09T18:53:35,288 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071)] 2024-12-09T18:53:35,288 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f80d7a9a346d7ab3b086bf0cb6c5225b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:35,288 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:35,288 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,288 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,290 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,291 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f80d7a9a346d7ab3b086bf0cb6c5225b columnFamilyName a 2024-12-09T18:53:35,291 DEBUG [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:35,292 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(327): Store=f80d7a9a346d7ab3b086bf0cb6c5225b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:35,292 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,293 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f80d7a9a346d7ab3b086bf0cb6c5225b columnFamilyName b 2024-12-09T18:53:35,293 DEBUG [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:35,294 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(327): Store=f80d7a9a346d7ab3b086bf0cb6c5225b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:35,294 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,295 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f80d7a9a346d7ab3b086bf0cb6c5225b columnFamilyName c 2024-12-09T18:53:35,295 DEBUG [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:35,296 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(327): Store=f80d7a9a346d7ab3b086bf0cb6c5225b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:35,296 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,297 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,298 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,299 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,299 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,299 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:35,301 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:35,302 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f80d7a9a346d7ab3b086bf0cb6c5225b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63913593, jitterRate=-0.047613248229026794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:35,303 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f80d7a9a346d7ab3b086bf0cb6c5225b: Writing region info on filesystem at 1733770415288Initializing all the Stores at 1733770415289 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770415289Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770415290 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770415290Cleaning up temporary data from old regions at 1733770415299 (+9 ms)Region opened successfully at 1733770415303 (+4 ms) 2024-12-09T18:53:35,307 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733770415306/Put/seqid=0 2024-12-09T18:53:35,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741881_1058 (size=4826) 2024-12-09T18:53:35,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741881_1058 (size=4826) 2024-12-09T18:53:35,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741881_1058 (size=4826) 2024-12-09T18:53:35,316 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41577/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in f80d7a9a346d7ab3b086bf0cb6c5225b/a 2024-12-09T18:53:35,325 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-09T18:53:35,325 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T18:53:35,326 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f80d7a9a346d7ab3b086bf0cb6c5225b: 2024-12-09T18:53:35,328 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/a/6a3cccdbc7954d2eae09d372324384eb_SeqId_3_ 2024-12-09T18:53:35,329 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41577/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into f80d7a9a346d7ab3b086bf0cb6c5225b/a as hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/a/6a3cccdbc7954d2eae09d372324384eb_SeqId_3_ - updating store file list. 2024-12-09T18:53:35,336 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6a3cccdbc7954d2eae09d372324384eb_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T18:53:35,336 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/a/6a3cccdbc7954d2eae09d372324384eb_SeqId_3_ into f80d7a9a346d7ab3b086bf0cb6c5225b/a 2024-12-09T18:53:35,336 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41577/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into f80d7a9a346d7ab3b086bf0cb6c5225b/a (new location: hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/a/6a3cccdbc7954d2eae09d372324384eb_SeqId_3_) 2024-12-09T18:53:35,374 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247, size=0 (0bytes) 2024-12-09T18:53:35,374 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 might be still open, length is 0 2024-12-09T18:53:35,374 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 2024-12-09T18:53:35,375 WARN [IPC Server handler 1 on default port 41577 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-09T18:53:35,375 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 after 1ms 2024-12-09T18:53:37,300 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:49766 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38029:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49766 dst: /127.0.0.1:38029 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38029 remote=/127.0.0.1:49766]. Total timeout mills is 60000, 58043 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:37,301 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:56678 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37407:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56678 dst: /127.0.0.1:37407 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:37,301 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:33112 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36059:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33112 dst: /127.0.0.1:36059 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:37,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741880_1059 (size=606) 2024-12-09T18:53:37,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741880_1059 (size=606) 2024-12-09T18:53:39,377 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 after 4002ms 2024-12-09T18:53:39,408 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:39,408 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 took 4034ms 2024-12-09T18:53:39,411 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247; continuing. 2024-12-09T18:53:39,411 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 so closing down 2024-12-09T18:53:39,411 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:39,413 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733770415247.temp 2024-12-09T18:53:39,436 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005-wal.1733770415247.temp 2024-12-09T18:53:39,436 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:39,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741882_1060 (size=329) 2024-12-09T18:53:39,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741882_1060 (size=329) 2024-12-09T18:53:39,443 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005-wal.1733770415247.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T18:53:39,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741882_1060 (size=329) 2024-12-09T18:53:39,445 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005-wal.1733770415247.temp to hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005 2024-12-09T18:53:39,445 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T18:53:39,445 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247, journal: Splitting hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247, size=0 (0bytes) at 1733770415374Finishing writing output for hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 so closing down at 1733770419411 (+4037 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005-wal.1733770415247.temp at 1733770419436 (+25 ms)3 split writer threads finished at 1733770419436Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005-wal.1733770415247.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733770419443 (+7 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005-wal.1733770415247.temp to hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005 at 1733770419445 (+2 ms)Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247, size=0, length=0, corrupted=false, cancelled=false at 1733770419445 2024-12-09T18:53:39,447 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 to hdfs://localhost:41577/hbase/oldWALs/wal.1733770415247 2024-12-09T18:53:39,448 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005 2024-12-09T18:53:39,448 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:39,450 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:39,463 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770419451, exclude list is [], retry=0 2024-12-09T18:53:39,466 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:39,466 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:39,466 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:39,489 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770419451 2024-12-09T18:53:39,490 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879)] 2024-12-09T18:53:39,490 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f80d7a9a346d7ab3b086bf0cb6c5225b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:39,490 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:39,490 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,490 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,492 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,493 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f80d7a9a346d7ab3b086bf0cb6c5225b columnFamilyName a 2024-12-09T18:53:39,493 DEBUG [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:39,500 DEBUG [StoreFileOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6a3cccdbc7954d2eae09d372324384eb_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T18:53:39,500 DEBUG [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/a/6a3cccdbc7954d2eae09d372324384eb_SeqId_3_ 2024-12-09T18:53:39,500 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(327): Store=f80d7a9a346d7ab3b086bf0cb6c5225b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:39,500 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,502 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f80d7a9a346d7ab3b086bf0cb6c5225b columnFamilyName b 2024-12-09T18:53:39,502 DEBUG [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:39,502 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(327): Store=f80d7a9a346d7ab3b086bf0cb6c5225b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:39,503 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,503 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f80d7a9a346d7ab3b086bf0cb6c5225b columnFamilyName c 2024-12-09T18:53:39,504 DEBUG [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:39,504 INFO [StoreOpener-f80d7a9a346d7ab3b086bf0cb6c5225b-1 {}] regionserver.HStore(327): Store=f80d7a9a346d7ab3b086bf0cb6c5225b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:39,504 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,505 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,507 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,508 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005 2024-12-09T18:53:39,531 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:39,532 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005 2024-12-09T18:53:39,532 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f80d7a9a346d7ab3b086bf0cb6c5225b 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-09T18:53:39,545 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/.tmp/a/105366da7b024829af1a344e51098bf7 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733770415342/Put/seqid=0 2024-12-09T18:53:39,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741884_1062 (size=5149) 2024-12-09T18:53:39,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741884_1062 (size=5149) 2024-12-09T18:53:39,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741884_1062 (size=5149) 2024-12-09T18:53:39,552 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/.tmp/a/105366da7b024829af1a344e51098bf7 2024-12-09T18:53:39,557 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/.tmp/a/105366da7b024829af1a344e51098bf7 as hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/a/105366da7b024829af1a344e51098bf7 2024-12-09T18:53:39,563 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/a/105366da7b024829af1a344e51098bf7, entries=1, sequenceid=5, filesize=5.0 K 2024-12-09T18:53:39,563 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for f80d7a9a346d7ab3b086bf0cb6c5225b in 31ms, sequenceid=5, compaction requested=false; wal=null 2024-12-09T18:53:39,564 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/0000000000000000005 2024-12-09T18:53:39,565 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,565 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,566 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:39,567 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f80d7a9a346d7ab3b086bf0cb6c5225b 2024-12-09T18:53:39,570 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/f80d7a9a346d7ab3b086bf0cb6c5225b/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-09T18:53:39,571 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f80d7a9a346d7ab3b086bf0cb6c5225b; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68871479, jitterRate=0.026265010237693787}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:39,571 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f80d7a9a346d7ab3b086bf0cb6c5225b: Writing region info on filesystem at 1733770419490Initializing all the Stores at 1733770419491 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770419492 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770419492Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770419492Obtaining lock to block concurrent updates at 1733770419532 (+40 ms)Preparing flush snapshotting stores in f80d7a9a346d7ab3b086bf0cb6c5225b at 1733770419532Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733770419532Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. at 1733770419532Flushing f80d7a9a346d7ab3b086bf0cb6c5225b/a: creating writer at 1733770419532Flushing f80d7a9a346d7ab3b086bf0cb6c5225b/a: appending metadata at 1733770419544 (+12 ms)Flushing f80d7a9a346d7ab3b086bf0cb6c5225b/a: closing flushed file at 1733770419544Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@119bd539: reopening flushed file at 1733770419556 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for f80d7a9a346d7ab3b086bf0cb6c5225b in 31ms, sequenceid=5, compaction requested=false; wal=null at 1733770419563 (+7 ms)Cleaning up temporary data from old regions at 1733770419565 (+2 ms)Region opened successfully at 1733770419571 (+6 ms) 2024-12-09T18:53:39,575 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f80d7a9a346d7ab3b086bf0cb6c5225b, disabling compactions & flushes 2024-12-09T18:53:39,575 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. 2024-12-09T18:53:39,575 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. 2024-12-09T18:53:39,575 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. after waiting 0 ms 2024-12-09T18:53:39,575 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. 2024-12-09T18:53:39,576 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733770415109.f80d7a9a346d7ab3b086bf0cb6c5225b. 2024-12-09T18:53:39,576 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f80d7a9a346d7ab3b086bf0cb6c5225b: Waiting for close lock at 1733770419575Disabling compacts and flushes for region at 1733770419575Disabling writes for close at 1733770419575Writing region close event to WAL at 1733770419576 (+1 ms)Closed at 1733770419576 2024-12-09T18:53:39,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741883_1061 (size=175) 2024-12-09T18:53:39,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741883_1061 (size=175) 2024-12-09T18:53:39,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741883_1061 (size=175) 2024-12-09T18:53:39,581 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:39,581 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733770419451) 2024-12-09T18:53:39,593 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=415 (was 413) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_286828688_22 at /127.0.0.1:56736 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_286828688_22 at /127.0.0.1:33168 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:41577 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:41577 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_286828688_22 at /127.0.0.1:49780 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=189 (was 197), ProcessCount=11 (was 11), AvailableMemoryMB=2891 (was 2937) 2024-12-09T18:53:39,606 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=415, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=189, ProcessCount=11, AvailableMemoryMB=2890 2024-12-09T18:53:39,621 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:39,625 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T18:53:39,629 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a437f8b9ba7d,35375,1733770403171 2024-12-09T18:53:39,630 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2798271 2024-12-09T18:53:39,631 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T18:53:39,633 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T18:53:39,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T18:53:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T18:53:39,644 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T18:53:39,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-09T18:53:39,646 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:39,648 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T18:53:39,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T18:53:39,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741885_1063 (size=694) 2024-12-09T18:53:39,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741885_1063 (size=694) 2024-12-09T18:53:39,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741885_1063 (size=694) 2024-12-09T18:53:39,661 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 58039aab64152816f1b072c38e494542, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8 2024-12-09T18:53:39,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741886_1064 (size=77) 2024-12-09T18:53:39,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741886_1064 (size=77) 2024-12-09T18:53:39,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741886_1064 (size=77) 2024-12-09T18:53:39,674 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:39,674 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 58039aab64152816f1b072c38e494542, disabling compactions & flushes 2024-12-09T18:53:39,674 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:39,674 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:39,674 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. after waiting 0 ms 2024-12-09T18:53:39,674 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:39,674 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:39,674 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 58039aab64152816f1b072c38e494542: Waiting for close lock at 1733770419674Disabling compacts and flushes for region at 1733770419674Disabling writes for close at 1733770419674Writing region close event to WAL at 1733770419674Closed at 1733770419674 2024-12-09T18:53:39,676 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T18:53:39,681 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733770419676"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733770419676"}]},"ts":"1733770419676"} 2024-12-09T18:53:39,685 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T18:53:39,688 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T18:53:39,691 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733770419688"}]},"ts":"1733770419688"} 2024-12-09T18:53:39,695 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-09T18:53:39,695 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a437f8b9ba7d=0} racks are {/default-rack=0} 2024-12-09T18:53:39,697 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T18:53:39,697 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T18:53:39,697 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T18:53:39,697 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T18:53:39,697 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T18:53:39,697 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T18:53:39,697 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T18:53:39,697 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T18:53:39,697 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T18:53:39,697 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T18:53:39,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, ASSIGN}] 2024-12-09T18:53:39,700 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, ASSIGN 2024-12-09T18:53:39,702 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, ASSIGN; state=OFFLINE, location=a437f8b9ba7d,46657,1733770404031; forceNewPlan=false, retain=false 2024-12-09T18:53:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T18:53:39,854 INFO [a437f8b9ba7d:35375 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T18:53:39,855 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=OPENING, regionLocation=a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:39,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, ASSIGN because future has completed 2024-12-09T18:53:39,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,46657,1733770404031}] 2024-12-09T18:53:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T18:53:40,017 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,017 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 58039aab64152816f1b072c38e494542, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:40,017 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,017 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:40,018 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,018 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,019 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,021 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf1 2024-12-09T18:53:40,021 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:40,021 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:40,021 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,023 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf2 2024-12-09T18:53:40,023 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:40,023 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:40,023 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,024 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,024 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,025 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,025 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,026 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T18:53:40,027 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,032 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:40,032 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 58039aab64152816f1b072c38e494542; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60492689, jitterRate=-0.09858869016170502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T18:53:40,032 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,033 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 58039aab64152816f1b072c38e494542: Running coprocessor pre-open hook at 1733770420018Writing region info on filesystem at 1733770420018Initializing all the Stores at 1733770420019 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770420019Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770420019Cleaning up temporary data from old regions at 1733770420025 (+6 ms)Running coprocessor post-open hooks at 1733770420032 (+7 ms)Region opened successfully at 1733770420033 (+1 ms) 2024-12-09T18:53:40,035 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., pid=6, masterSystemTime=1733770420011 2024-12-09T18:53:40,037 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,037 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,038 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=OPEN, openSeqNum=2, regionLocation=a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:40,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,46657,1733770404031 because future has completed 2024-12-09T18:53:40,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T18:53:40,047 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,46657,1733770404031 in 184 msec 2024-12-09T18:53:40,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T18:53:40,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, ASSIGN in 348 msec 2024-12-09T18:53:40,051 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T18:53:40,051 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733770420051"}]},"ts":"1733770420051"} 2024-12-09T18:53:40,054 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-09T18:53:40,055 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T18:53:40,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 417 msec 2024-12-09T18:53:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T18:53:40,277 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-09T18:53:40,277 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-09T18:53:40,278 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T18:53:40,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-09T18:53:40,284 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T18:53:40,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-09T18:53:40,298 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,46657,1733770404031, seqNum=2] 2024-12-09T18:53:40,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=58039aab64152816f1b072c38e494542, source=a437f8b9ba7d,46657,1733770404031, destination=a437f8b9ba7d,35449,1733770403924, warming up region on a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T18:53:40,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=58039aab64152816f1b072c38e494542, source=a437f8b9ba7d,46657,1733770404031, destination=a437f8b9ba7d,35449,1733770403924, running balancer 2024-12-09T18:53:40,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45553, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T18:53:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE 2024-12-09T18:53:40,317 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE 2024-12-09T18:53:40,319 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=CLOSING, regionLocation=a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:40,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] regionserver.HRegion(7855): Warmup {ENCODED => 58039aab64152816f1b072c38e494542, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:40,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE because future has completed 2024-12-09T18:53:40,322 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,322 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T18:53:40,322 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,46657,1733770404031}] 2024-12-09T18:53:40,323 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf1 2024-12-09T18:53:40,323 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:40,324 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:40,324 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,325 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf2 2024-12-09T18:53:40,325 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:40,326 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] regionserver.HRegion(1722): Closing 58039aab64152816f1b072c38e494542, disabling compactions & flushes 2024-12-09T18:53:40,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. after waiting 0 ms 2024-12-09T18:53:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] regionserver.HRegion(1676): Region close journal for 58039aab64152816f1b072c38e494542: Waiting for close lock at 1733770420326Disabling compacts and flushes for region at 1733770420326Disabling writes for close at 1733770420326Writing region close event to WAL at 1733770420327 (+1 ms)Closed at 1733770420327 2024-12-09T18:53:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-09T18:53:40,483 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,483 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T18:53:40,484 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 58039aab64152816f1b072c38e494542, disabling compactions & flushes 2024-12-09T18:53:40,485 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,485 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,485 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. after waiting 0 ms 2024-12-09T18:53:40,485 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,485 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 58039aab64152816f1b072c38e494542 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-09T18:53:40,507 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf1/79dead2f3a794d249ef8460bc8c908ed is 35, key is r1/cf1:q/1733770420300/Put/seqid=0 2024-12-09T18:53:40,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741887_1065 (size=4783) 2024-12-09T18:53:40,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741887_1065 (size=4783) 2024-12-09T18:53:40,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741887_1065 (size=4783) 2024-12-09T18:53:40,513 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf1/79dead2f3a794d249ef8460bc8c908ed 2024-12-09T18:53:40,521 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf1/79dead2f3a794d249ef8460bc8c908ed as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/79dead2f3a794d249ef8460bc8c908ed 2024-12-09T18:53:40,527 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/79dead2f3a794d249ef8460bc8c908ed, entries=1, sequenceid=5, filesize=4.7 K 2024-12-09T18:53:40,528 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 58039aab64152816f1b072c38e494542 in 43ms, sequenceid=5, compaction requested=false 2024-12-09T18:53:40,528 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-09T18:53:40,533 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T18:53:40,535 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,535 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 58039aab64152816f1b072c38e494542: Waiting for close lock at 1733770420484Running coprocessor pre-close hooks at 1733770420484Disabling compacts and flushes for region at 1733770420484Disabling writes for close at 1733770420485 (+1 ms)Obtaining lock to block concurrent updates at 1733770420485Preparing flush snapshotting stores in 58039aab64152816f1b072c38e494542 at 1733770420485Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733770420486 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. at 1733770420487 (+1 ms)Flushing 58039aab64152816f1b072c38e494542/cf1: creating writer at 1733770420487Flushing 58039aab64152816f1b072c38e494542/cf1: appending metadata at 1733770420507 (+20 ms)Flushing 58039aab64152816f1b072c38e494542/cf1: closing flushed file at 1733770420507Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d5cce31: reopening flushed file at 1733770420520 (+13 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 58039aab64152816f1b072c38e494542 in 43ms, sequenceid=5, compaction requested=false at 1733770420528 (+8 ms)Writing region close event to WAL at 1733770420529 (+1 ms)Running coprocessor post-close hooks at 1733770420534 (+5 ms)Closed at 1733770420535 (+1 ms) 2024-12-09T18:53:40,536 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 58039aab64152816f1b072c38e494542 move to a437f8b9ba7d,35449,1733770403924 record at close sequenceid=5 2024-12-09T18:53:40,539 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,540 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=CLOSED 2024-12-09T18:53:40,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,46657,1733770404031 because future has completed 2024-12-09T18:53:40,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T18:53:40,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,46657,1733770404031 in 222 msec 2024-12-09T18:53:40,550 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE; state=CLOSED, location=a437f8b9ba7d,35449,1733770403924; forceNewPlan=false, retain=false 2024-12-09T18:53:40,701 INFO [a437f8b9ba7d:35375 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T18:53:40,701 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=OPENING, regionLocation=a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:40,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE because future has completed 2024-12-09T18:53:40,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,35449,1733770403924}] 2024-12-09T18:53:40,865 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,865 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 58039aab64152816f1b072c38e494542, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:40,865 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,865 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:40,865 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,865 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,867 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,868 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf1 2024-12-09T18:53:40,868 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:40,875 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/79dead2f3a794d249ef8460bc8c908ed 2024-12-09T18:53:40,875 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:40,876 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,877 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf2 2024-12-09T18:53:40,877 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:40,878 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:40,878 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,879 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,880 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,881 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,881 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,882 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T18:53:40,884 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,885 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 58039aab64152816f1b072c38e494542; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67182295, jitterRate=0.0010942071676254272}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T18:53:40,885 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:40,886 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 58039aab64152816f1b072c38e494542: Running coprocessor pre-open hook at 1733770420865Writing region info on filesystem at 1733770420865Initializing all the Stores at 1733770420867 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770420867Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770420867Cleaning up temporary data from old regions at 1733770420881 (+14 ms)Running coprocessor post-open hooks at 1733770420885 (+4 ms)Region opened successfully at 1733770420886 (+1 ms) 2024-12-09T18:53:40,887 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., pid=9, masterSystemTime=1733770420860 2024-12-09T18:53:40,889 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,890 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:40,890 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=OPEN, openSeqNum=9, regionLocation=a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:40,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,35449,1733770403924 because future has completed 2024-12-09T18:53:40,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-09T18:53:40,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,35449,1733770403924 in 187 msec 2024-12-09T18:53:40,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE in 582 msec 2024-12-09T18:53:40,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T18:53:40,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T18:53:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:38526 deadline: 1733770480929, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a437f8b9ba7d port=35449 startCode=1733770403924. As of locationSeqNum=5. 2024-12-09T18:53:40,955 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,46657,1733770404031, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,46657,1733770404031, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a437f8b9ba7d port=35449 startCode=1733770403924. As of locationSeqNum=5. 2024-12-09T18:53:40,955 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,46657,1733770404031, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a437f8b9ba7d port=35449 startCode=1733770403924. As of locationSeqNum=5. 2024-12-09T18:53:40,956 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,46657,1733770404031, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,35449,1733770403924, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a437f8b9ba7d port=35449 startCode=1733770403924. As of locationSeqNum=5. 2024-12-09T18:53:41,065 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T18:53:41,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T18:53:41,078 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 58039aab64152816f1b072c38e494542 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-09T18:53:41,093 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf1/cce79ccbb2094541838749b615c25bc3 is 29, key is r1/cf1:/1733770421069/DeleteFamily/seqid=0 2024-12-09T18:53:41,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741888_1066 (size=4906) 2024-12-09T18:53:41,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741888_1066 (size=4906) 2024-12-09T18:53:41,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741888_1066 (size=4906) 2024-12-09T18:53:41,102 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf1/cce79ccbb2094541838749b615c25bc3 2024-12-09T18:53:41,108 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cce79ccbb2094541838749b615c25bc3 2024-12-09T18:53:41,122 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf2/acb27138ef5f4adfaebc5fdedcd2a9b3 is 29, key is r1/cf2:/1733770421069/DeleteFamily/seqid=0 2024-12-09T18:53:41,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741889_1067 (size=4906) 2024-12-09T18:53:41,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741889_1067 (size=4906) 2024-12-09T18:53:41,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741889_1067 (size=4906) 2024-12-09T18:53:41,133 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf2/acb27138ef5f4adfaebc5fdedcd2a9b3 2024-12-09T18:53:41,139 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for acb27138ef5f4adfaebc5fdedcd2a9b3 2024-12-09T18:53:41,141 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf1/cce79ccbb2094541838749b615c25bc3 as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/cce79ccbb2094541838749b615c25bc3 2024-12-09T18:53:41,147 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cce79ccbb2094541838749b615c25bc3 2024-12-09T18:53:41,147 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/cce79ccbb2094541838749b615c25bc3, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T18:53:41,149 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf2/acb27138ef5f4adfaebc5fdedcd2a9b3 as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/acb27138ef5f4adfaebc5fdedcd2a9b3 2024-12-09T18:53:41,154 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for acb27138ef5f4adfaebc5fdedcd2a9b3 2024-12-09T18:53:41,154 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/acb27138ef5f4adfaebc5fdedcd2a9b3, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T18:53:41,155 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 58039aab64152816f1b072c38e494542 in 77ms, sequenceid=12, compaction requested=false 2024-12-09T18:53:41,156 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 58039aab64152816f1b072c38e494542: 2024-12-09T18:53:41,158 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T18:53:41,159 DEBUG [Time-limited test {}] regionserver.HStore(1541): 58039aab64152816f1b072c38e494542/cf1 is initiating major compaction (all files) 2024-12-09T18:53:41,159 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T18:53:41,159 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:41,159 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 58039aab64152816f1b072c38e494542/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,160 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/79dead2f3a794d249ef8460bc8c908ed, hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/cce79ccbb2094541838749b615c25bc3] into tmpdir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp, totalSize=9.5 K 2024-12-09T18:53:41,161 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 79dead2f3a794d249ef8460bc8c908ed, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733770420300 2024-12-09T18:53:41,161 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cce79ccbb2094541838749b615c25bc3, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T18:53:41,172 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 58039aab64152816f1b072c38e494542#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T18:53:41,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741890_1068 (size=4626) 2024-12-09T18:53:41,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741890_1068 (size=4626) 2024-12-09T18:53:41,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741890_1068 (size=4626) 2024-12-09T18:53:41,187 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf1/e92e3827dea44bb9a5a9076577f710a4 as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/e92e3827dea44bb9a5a9076577f710a4 2024-12-09T18:53:41,203 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 58039aab64152816f1b072c38e494542/cf1 of 58039aab64152816f1b072c38e494542 into e92e3827dea44bb9a5a9076577f710a4(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T18:53:41,203 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 58039aab64152816f1b072c38e494542: 2024-12-09T18:53:41,203 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-09T18:53:41,204 DEBUG [Time-limited test {}] regionserver.HStore(1541): 58039aab64152816f1b072c38e494542/cf2 is initiating major compaction (all files) 2024-12-09T18:53:41,204 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T18:53:41,204 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T18:53:41,204 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 58039aab64152816f1b072c38e494542/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,204 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/acb27138ef5f4adfaebc5fdedcd2a9b3] into tmpdir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp, totalSize=4.8 K 2024-12-09T18:53:41,205 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting acb27138ef5f4adfaebc5fdedcd2a9b3, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T18:53:41,212 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 58039aab64152816f1b072c38e494542#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T18:53:41,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741891_1069 (size=4592) 2024-12-09T18:53:41,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741891_1069 (size=4592) 2024-12-09T18:53:41,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741891_1069 (size=4592) 2024-12-09T18:53:41,229 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/.tmp/cf2/7697bbc23fb240f7a4d971818d3e69fa as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/7697bbc23fb240f7a4d971818d3e69fa 2024-12-09T18:53:41,239 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 58039aab64152816f1b072c38e494542/cf2 of 58039aab64152816f1b072c38e494542 into 7697bbc23fb240f7a4d971818d3e69fa(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T18:53:41,239 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 58039aab64152816f1b072c38e494542: 2024-12-09T18:53:41,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=58039aab64152816f1b072c38e494542, source=a437f8b9ba7d,35449,1733770403924, destination=a437f8b9ba7d,46657,1733770404031, warming up region on a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:41,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=58039aab64152816f1b072c38e494542, source=a437f8b9ba7d,35449,1733770403924, destination=a437f8b9ba7d,46657,1733770404031, running balancer 2024-12-09T18:53:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE 2024-12-09T18:53:41,244 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE 2024-12-09T18:53:41,246 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=CLOSING, regionLocation=a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:41,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] regionserver.HRegion(7855): Warmup {ENCODED => 58039aab64152816f1b072c38e494542, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:41,247 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,248 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf1 2024-12-09T18:53:41,248 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:41,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE because future has completed 2024-12-09T18:53:41,249 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T18:53:41,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,35449,1733770403924}] 2024-12-09T18:53:41,256 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/79dead2f3a794d249ef8460bc8c908ed 2024-12-09T18:53:41,260 INFO [StoreFileOpener-58039aab64152816f1b072c38e494542-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cce79ccbb2094541838749b615c25bc3 2024-12-09T18:53:41,261 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/cce79ccbb2094541838749b615c25bc3 2024-12-09T18:53:41,266 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/e92e3827dea44bb9a5a9076577f710a4 2024-12-09T18:53:41,266 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:41,266 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,267 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf2 2024-12-09T18:53:41,267 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:41,273 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/7697bbc23fb240f7a4d971818d3e69fa 2024-12-09T18:53:41,277 INFO [StoreFileOpener-58039aab64152816f1b072c38e494542-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for acb27138ef5f4adfaebc5fdedcd2a9b3 2024-12-09T18:53:41,278 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/acb27138ef5f4adfaebc5fdedcd2a9b3 2024-12-09T18:53:41,278 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] regionserver.HRegion(1722): Closing 58039aab64152816f1b072c38e494542, disabling compactions & flushes 2024-12-09T18:53:41,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. after waiting 0 ms 2024-12-09T18:53:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46657 {}] regionserver.HRegion(1676): Region close journal for 58039aab64152816f1b072c38e494542: Waiting for close lock at 1733770421278Disabling compacts and flushes for region at 1733770421278Disabling writes for close at 1733770421278Writing region close event to WAL at 1733770421280 (+2 ms)Closed at 1733770421280 2024-12-09T18:53:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-09T18:53:41,403 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,403 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T18:53:41,403 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 58039aab64152816f1b072c38e494542, disabling compactions & flushes 2024-12-09T18:53:41,403 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,403 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,403 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. after waiting 0 ms 2024-12-09T18:53:41,403 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,404 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/79dead2f3a794d249ef8460bc8c908ed, hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/cce79ccbb2094541838749b615c25bc3] to archive 2024-12-09T18:53:41,406 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T18:53:41,410 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/79dead2f3a794d249ef8460bc8c908ed to hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/79dead2f3a794d249ef8460bc8c908ed 2024-12-09T18:53:41,412 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/cce79ccbb2094541838749b615c25bc3 to hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/cce79ccbb2094541838749b615c25bc3 2024-12-09T18:53:41,428 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/acb27138ef5f4adfaebc5fdedcd2a9b3] to archive 2024-12-09T18:53:41,429 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T18:53:41,431 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/acb27138ef5f4adfaebc5fdedcd2a9b3 to hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/acb27138ef5f4adfaebc5fdedcd2a9b3 2024-12-09T18:53:41,436 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-09T18:53:41,437 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,437 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 58039aab64152816f1b072c38e494542: Waiting for close lock at 1733770421403Running coprocessor pre-close hooks at 1733770421403Disabling compacts and flushes for region at 1733770421403Disabling writes for close at 1733770421403Writing region close event to WAL at 1733770421433 (+30 ms)Running coprocessor post-close hooks at 1733770421437 (+4 ms)Closed at 1733770421437 2024-12-09T18:53:41,437 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 58039aab64152816f1b072c38e494542 move to a437f8b9ba7d,46657,1733770404031 record at close sequenceid=12 2024-12-09T18:53:41,439 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,440 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=CLOSED 2024-12-09T18:53:41,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,35449,1733770403924 because future has completed 2024-12-09T18:53:41,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T18:53:41,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,35449,1733770403924 in 195 msec 2024-12-09T18:53:41,449 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE; state=CLOSED, location=a437f8b9ba7d,46657,1733770404031; forceNewPlan=false, retain=false 2024-12-09T18:53:41,600 INFO [a437f8b9ba7d:35375 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T18:53:41,600 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=OPENING, regionLocation=a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:41,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE because future has completed 2024-12-09T18:53:41,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,46657,1733770404031}] 2024-12-09T18:53:41,766 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,767 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 58039aab64152816f1b072c38e494542, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:41,767 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,767 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:41,768 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,768 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,770 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,771 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf1 2024-12-09T18:53:41,771 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:41,780 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/e92e3827dea44bb9a5a9076577f710a4 2024-12-09T18:53:41,780 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:41,780 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,782 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf2 2024-12-09T18:53:41,782 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:41,787 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/7697bbc23fb240f7a4d971818d3e69fa 2024-12-09T18:53:41,787 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:41,788 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,788 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,790 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,790 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,791 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,791 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T18:53:41,793 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,794 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 58039aab64152816f1b072c38e494542; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59616584, jitterRate=-0.11164367198944092}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T18:53:41,794 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,794 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 58039aab64152816f1b072c38e494542: Running coprocessor pre-open hook at 1733770421768Writing region info on filesystem at 1733770421768Initializing all the Stores at 1733770421769 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770421769Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770421769Cleaning up temporary data from old regions at 1733770421791 (+22 ms)Running coprocessor post-open hooks at 1733770421794 (+3 ms)Region opened successfully at 1733770421794 2024-12-09T18:53:41,795 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., pid=12, masterSystemTime=1733770421759 2024-12-09T18:53:41,797 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,798 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,798 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=OPEN, openSeqNum=18, regionLocation=a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:41,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,46657,1733770404031 because future has completed 2024-12-09T18:53:41,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-09T18:53:41,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,46657,1733770404031 in 196 msec 2024-12-09T18:53:41,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, REOPEN/MOVE in 561 msec 2024-12-09T18:53:41,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T18:53:41,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T18:53:41,849 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server a437f8b9ba7d,46657,1733770404031: testing ***** 2024-12-09T18:53:41,850 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T18:53:41,851 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T18:53:41,852 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T18:53:41,854 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T18:53:41,855 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T18:53:41,862 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 471598080 }, "NonHeapMemoryUsage": { "committed": 171966464, "init": 7667712, "max": -1, "used": 169342520 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "a437f8b9ba7d", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 28, "ProcessCallTime_mean": 7, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 14, "ProcessCallTime_75th_percentile": 21, "ProcessCallTime_90th_percentile": 25, "ProcessCallTime_95th_percentile": 26, "ProcessCallTime_98th_percentile": 27, "ProcessCallTime_99th_percentile": 27, "ProcessCallTime_99.9th_percentile": 27, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 28, "TotalCallTime_mean": 7, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 14, "TotalCallTime_75th_percentile": 21, "TotalCallTime_90th_percentile": 25, "TotalCallTime_95th_percentile": 26, "TotalCallTime_98th_percentile": 27, "TotalCallTime_99th_percentile": 27, "TotalCallTime_99.9th_percentile": 27, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "a437f8b9ba7d", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:52975", "tag.serverName": "a437f8b9ba7d,37501,1733770404084", "tag.clusterId": "09f08c9b-bbeb-413c-9bb5-a0da44c958b7", "tag.Context": "regionserver", "tag.Hostname": "a437f8b9ba7d", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733770404084, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T18:53:41,864 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35375 {}] master.MasterRpcServices(700): a437f8b9ba7d,46657,1733770404031 reported a fatal error: ***** ABORTING region server a437f8b9ba7d,46657,1733770404031: testing ***** 2024-12-09T18:53:41,866 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a437f8b9ba7d,46657,1733770404031' ***** 2024-12-09T18:53:41,866 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-09T18:53:41,866 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T18:53:41,866 INFO [RS:1;a437f8b9ba7d:46657 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-09T18:53:41,866 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T18:53:41,866 INFO [RS:1;a437f8b9ba7d:46657 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-09T18:53:41,867 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(3091): Received CLOSE for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35449 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:32874 deadline: 1733770481867, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a437f8b9ba7d port=46657 startCode=1733770404031. As of locationSeqNum=12. 2024-12-09T18:53:41,867 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(956): aborting server a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:41,867 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T18:53:41,867 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 58039aab64152816f1b072c38e494542, disabling compactions & flushes 2024-12-09T18:53:41,867 INFO [RS:1;a437f8b9ba7d:46657 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a437f8b9ba7d:46657. 2024-12-09T18:53:41,867 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,867 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,35449,1733770403924, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,35449,1733770403924, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a437f8b9ba7d port=46657 startCode=1733770404031. As of locationSeqNum=12. 2024-12-09T18:53:41,867 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,868 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,35449,1733770403924, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a437f8b9ba7d port=46657 startCode=1733770404031. As of locationSeqNum=12. 2024-12-09T18:53:41,868 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. after waiting 0 ms 2024-12-09T18:53:41,868 DEBUG [RS:1;a437f8b9ba7d:46657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T18:53:41,868 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,868 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,35449,1733770403924, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,46657,1733770404031, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a437f8b9ba7d port=46657 startCode=1733770404031. As of locationSeqNum=12. 2024-12-09T18:53:41,868 DEBUG [RS:1;a437f8b9ba7d:46657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T18:53:41,868 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T18:53:41,869 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T18:53:41,869 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T18:53:41,869 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T18:53:41,869 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a437f8b9ba7d,46657,1733770404031 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:41,869 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-09T18:53:41,869 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 58039aab64152816f1b072c38e494542: Waiting for close lock at 1733770421867Running coprocessor pre-close hooks at 1733770421867Disabling compacts and flushes for region at 1733770421867Disabling writes for close at 1733770421868 (+1 ms)Writing region close event to WAL at 1733770421869 (+1 ms)Running coprocessor post-close hooks at 1733770421869Closed at 1733770421869 2024-12-09T18:53:41,869 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T18:53:41,869 DEBUG [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T18:53:41,869 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T18:53:41,869 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:41,869 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T18:53:41,869 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T18:53:41,869 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T18:53:41,870 DEBUG [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T18:53:41,870 ERROR [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-09T18:53:41,870 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T18:53:41,870 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T18:53:41,870 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a437f8b9ba7d,46657,1733770404031 aborting 2024-12-09T18:53:41,870 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733770421869Running coprocessor pre-close hooks at 1733770421869Disabling compacts and flushes for region at 1733770421869Disabling writes for close at 1733770421869Writing region close event to WAL at 1733770421870 (+1 ms)Running coprocessor post-close hooks at 1733770421870Closed at 1733770421870 2024-12-09T18:53:41,870 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a437f8b9ba7d,46657,1733770404031 aborting 2024-12-09T18:53:41,870 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 from cache 2024-12-09T18:53:41,870 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T18:53:41,889 INFO [regionserver/a437f8b9ba7d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T18:53:41,975 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T18:53:41,978 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1] 2024-12-09T18:53:41,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a437f8b9ba7d,46657,1733770404031 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:41,981 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a437f8b9ba7d,46657,1733770404031 aborting 2024-12-09T18:53:41,982 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a437f8b9ba7d,46657,1733770404031 aborting 2024-12-09T18:53:41,982 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 from cache 2024-12-09T18:53:42,070 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(976): stopping server a437f8b9ba7d,46657,1733770404031; all regions closed. 2024-12-09T18:53:42,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741836_1012 (size=7323) 2024-12-09T18:53:42,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741836_1012 (size=7323) 2024-12-09T18:53:42,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741836_1012 (size=7323) 2024-12-09T18:53:42,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741835_1011 (size=2288) 2024-12-09T18:53:42,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741835_1011 (size=2288) 2024-12-09T18:53:42,081 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845 not finished, retry = 0 2024-12-09T18:53:42,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741835_1011 (size=2288) 2024-12-09T18:53:42,182 DEBUG [RS:1;a437f8b9ba7d:46657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T18:53:42,182 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T18:53:42,183 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T18:53:42,183 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.ChoreService(370): Chore service for: regionserver/a437f8b9ba7d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T18:53:42,183 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T18:53:42,183 INFO [regionserver/a437f8b9ba7d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T18:53:42,184 INFO [RS:1;a437f8b9ba7d:46657 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46657 2024-12-09T18:53:42,185 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T18:53:42,186 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1] 2024-12-09T18:53:42,190 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Call to address=a437f8b9ba7d:46657 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-09T18:53:42,190 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 is org.apache.hadoop.hbase.exceptions.ConnectionClosedException: Connection closed 2024-12-09T18:53:42,190 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 from cache 2024-12-09T18:53:42,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T18:53:42,229 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a437f8b9ba7d,46657,1733770404031 2024-12-09T18:53:42,229 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T18:53:42,231 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a437f8b9ba7d,46657,1733770404031] 2024-12-09T18:53:42,245 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a437f8b9ba7d,46657,1733770404031 already deleted, retry=false 2024-12-09T18:53:42,245 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a437f8b9ba7d,46657,1733770404031 on a437f8b9ba7d,35375,1733770403171 2024-12-09T18:53:42,252 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a437f8b9ba7d,46657,1733770404031, splitWal=true, meta=true 2024-12-09T18:53:42,256 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a437f8b9ba7d,46657,1733770404031, splitWal=true, meta=true 2024-12-09T18:53:42,256 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for a437f8b9ba7d,46657,1733770404031 (carryingMeta=true) a437f8b9ba7d,46657,1733770404031/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1be340d7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T18:53:42,258 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a437f8b9ba7d,46657,1733770404031, splitWal=true, meta=true, isMeta: true 2024-12-09T18:53:42,260 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting 2024-12-09T18:53:42,261 INFO [PEWorker-2 {}] master.SplitWALManager(105): a437f8b9ba7d,46657,1733770404031 WAL count=1, meta=true 2024-12-09T18:53:42,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta}] 2024-12-09T18:53:42,267 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:42,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta, worker=a437f8b9ba7d,37501,1733770404084}] 2024-12-09T18:53:42,337 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T18:53:42,338 DEBUG [pool-70-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46657-0x1000c23f6fa0002, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T18:53:42,338 INFO [RS:1;a437f8b9ba7d:46657 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T18:53:42,338 INFO [RS:1;a437f8b9ba7d:46657 {}] regionserver.HRegionServer(1031): Exiting; stopping=a437f8b9ba7d,46657,1733770404031; zookeeper connection closed. 2024-12-09T18:53:42,338 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@29590737 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@29590737 2024-12-09T18:53:42,428 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T18:53:42,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45431, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T18:53:42,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37501 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-09T18:53:42,447 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta, size=7.2 K (7323bytes) 2024-12-09T18:53:42,447 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta 2024-12-09T18:53:42,448 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta after 1ms 2024-12-09T18:53:42,471 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:42,471 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta took 24ms 2024-12-09T18:53:42,478 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-09T18:53:42,478 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta so closing down 2024-12-09T18:53:42,479 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:42,480 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta.temp 2024-12-09T18:53:42,495 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T18:53:42,496 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1] 2024-12-09T18:53:42,498 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a437f8b9ba7d:46657 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a437f8b9ba7d/172.17.0.2:46657 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:42,499 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1, error=java.net.ConnectException: Call to address=a437f8b9ba7d:46657 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a437f8b9ba7d/172.17.0.2:46657 2024-12-09T18:53:42,499 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-09T18:53:42,499 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 from cache 2024-12-09T18:53:42,500 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address a437f8b9ba7d:46657 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a437f8b9ba7d/172.17.0.2:46657 2024-12-09T18:53:42,505 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta.temp 2024-12-09T18:53:42,506 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:42,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741892_1070 (size=7071) 2024-12-09T18:53:42,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741892_1070 (size=7071) 2024-12-09T18:53:42,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741892_1070 (size=7071) 2024-12-09T18:53:42,515 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-09T18:53:42,517 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta.temp to hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-09T18:53:42,517 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 45 ms; skipped=1; WAL=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-12-09T18:53:42,517 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta, journal: Splitting hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta, size=7.2 K (7323bytes) at 1733770422447Finishing writing output for hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta so closing down at 1733770422478 (+31 ms)Creating recovered edits writer path=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta.temp at 1733770422505 (+27 ms)3 split writer threads finished at 1733770422506 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733770422515 (+9 ms)Rename recovered edits hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta.temp to hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733770422517 (+2 ms)Processed 16 edits across 1 Regions in 45 ms; skipped=1; WAL=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1733770422517 2024-12-09T18:53:42,517 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta 2024-12-09T18:53:42,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-09T18:53:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-09T18:53:42,525 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta to hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/oldWALs 2024-12-09T18:53:42,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T18:53:42,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta, worker=a437f8b9ba7d,37501,1733770404084 in 256 msec 2024-12-09T18:53:42,528 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:42,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T18:53:42,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure a437f8b9ba7d%2C46657%2C1733770404031.meta.1733770406327.meta, worker=a437f8b9ba7d,37501,1733770404084 in 266 msec 2024-12-09T18:53:42,532 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting dir is empty, no logs to split. 2024-12-09T18:53:42,532 INFO [PEWorker-2 {}] master.SplitWALManager(105): a437f8b9ba7d,46657,1733770404031 WAL count=0, meta=true 2024-12-09T18:53:42,532 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if a437f8b9ba7d,46657,1733770404031 WAL splitting is done? wals=0, meta=true 2024-12-09T18:53:42,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T18:53:42,536 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T18:53:42,537 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T18:53:42,687 DEBUG [a437f8b9ba7d:35375 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-09T18:53:42,688 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(204): Hosts are {a437f8b9ba7d=0} racks are {/default-rack=0} 2024-12-09T18:53:42,688 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T18:53:42,688 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T18:53:42,688 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T18:53:42,688 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T18:53:42,688 INFO [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T18:53:42,688 INFO [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T18:53:42,688 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T18:53:42,689 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:42,690 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a437f8b9ba7d,37501,1733770404084, state=OPENING 2024-12-09T18:53:42,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T18:53:42,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T18:53:42,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T18:53:42,729 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T18:53:42,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:42,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:42,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a437f8b9ba7d,37501,1733770404084}] 2024-12-09T18:53:42,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:42,888 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T18:53:42,888 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:42,888 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T18:53:42,890 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a437f8b9ba7d%2C37501%2C1733770404084.meta, suffix=.meta, logDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,37501,1733770404084, archiveDir=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/oldWALs, maxLogs=32 2024-12-09T18:53:42,902 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,37501,1733770404084/a437f8b9ba7d%2C37501%2C1733770404084.meta.1733770422890.meta, exclude list is [], retry=0 2024-12-09T18:53:42,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:42,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:42,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:42,927 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,37501,1733770404084/a437f8b9ba7d%2C37501%2C1733770404084.meta.1733770422890.meta 2024-12-09T18:53:42,927 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:36879:36879)] 2024-12-09T18:53:42,928 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:42,928 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T18:53:42,928 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T18:53:42,928 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T18:53:42,928 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T18:53:42,928 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:42,929 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T18:53:42,929 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T18:53:42,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T18:53:42,931 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T18:53:42,931 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:42,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:42,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T18:53:42,933 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T18:53:42,933 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:42,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:42,933 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T18:53:42,934 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T18:53:42,934 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:42,935 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:42,935 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T18:53:42,935 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T18:53:42,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:42,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T18:53:42,936 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T18:53:42,937 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740 2024-12-09T18:53:42,938 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740 2024-12-09T18:53:42,939 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-09T18:53:42,966 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:42,969 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-09T18:53:42,969 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-09T18:53:42,985 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/info/a8ebe20099644199a33884329e6ed15a is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542./info:regioninfo/1733770421798/Put/seqid=0 2024-12-09T18:53:42,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741894_1072 (size=11177) 2024-12-09T18:53:42,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741894_1072 (size=11177) 2024-12-09T18:53:42,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741894_1072 (size=11177) 2024-12-09T18:53:42,991 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/info/a8ebe20099644199a33884329e6ed15a 2024-12-09T18:53:43,005 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T18:53:43,006 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,37501,1733770404084, seqNum=-1] 2024-12-09T18:53:43,006 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T18:53:43,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T18:53:43,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37501 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:50286 deadline: 1733770483009, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:43,010 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,37501,1733770404084, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,37501,1733770404084, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on a437f8b9ba7d,37501,1733770404084 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T18:53:43,010 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,37501,1733770404084, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on a437f8b9ba7d,37501,1733770404084 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T18:53:43,010 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,37501,1733770404084, seqNum=-1 from cache 2024-12-09T18:53:43,013 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/ns/08bb6aaa6f064095b1590902f60a97a5 is 43, key is default/ns:d/1733770406559/Put/seqid=0 2024-12-09T18:53:43,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741895_1073 (size=5153) 2024-12-09T18:53:43,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741895_1073 (size=5153) 2024-12-09T18:53:43,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741895_1073 (size=5153) 2024-12-09T18:53:43,022 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/ns/08bb6aaa6f064095b1590902f60a97a5 2024-12-09T18:53:43,045 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/table/718c46a39d1248f9ad4b06740dbf9a03 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733770420051/Put/seqid=0 2024-12-09T18:53:43,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741896_1074 (size=5431) 2024-12-09T18:53:43,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741896_1074 (size=5431) 2024-12-09T18:53:43,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741896_1074 (size=5431) 2024-12-09T18:53:43,051 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/table/718c46a39d1248f9ad4b06740dbf9a03 2024-12-09T18:53:43,059 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/info/a8ebe20099644199a33884329e6ed15a as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/info/a8ebe20099644199a33884329e6ed15a 2024-12-09T18:53:43,067 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/info/a8ebe20099644199a33884329e6ed15a, entries=36, sequenceid=18, filesize=10.9 K 2024-12-09T18:53:43,069 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/ns/08bb6aaa6f064095b1590902f60a97a5 as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/ns/08bb6aaa6f064095b1590902f60a97a5 2024-12-09T18:53:43,076 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/ns/08bb6aaa6f064095b1590902f60a97a5, entries=2, sequenceid=18, filesize=5.0 K 2024-12-09T18:53:43,078 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/table/718c46a39d1248f9ad4b06740dbf9a03 as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/table/718c46a39d1248f9ad4b06740dbf9a03 2024-12-09T18:53:43,084 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/table/718c46a39d1248f9ad4b06740dbf9a03, entries=2, sequenceid=18, filesize=5.3 K 2024-12-09T18:53:43,084 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 115ms, sequenceid=18, compaction requested=false; wal=null 2024-12-09T18:53:43,085 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-09T18:53:43,086 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T18:53:43,086 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T18:53:43,087 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T18:53:43,089 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T18:53:43,091 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-09T18:53:43,092 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71978428, jitterRate=0.07256215810775757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T18:53:43,092 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T18:53:43,092 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733770422929Writing region info on filesystem at 1733770422929Initializing all the Stores at 1733770422930 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733770422930Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733770422930Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770422930Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733770422930Obtaining lock to block concurrent updates at 1733770422969 (+39 ms)Preparing flush snapshotting stores in 1588230740 at 1733770422969Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733770422969Flushing stores of hbase:meta,,1.1588230740 at 1733770422969Flushing 1588230740/info: creating writer at 1733770422969Flushing 1588230740/info: appending metadata at 1733770422984 (+15 ms)Flushing 1588230740/info: closing flushed file at 1733770422984Flushing 1588230740/ns: creating writer at 1733770422998 (+14 ms)Flushing 1588230740/ns: appending metadata at 1733770423012 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733770423012Flushing 1588230740/table: creating writer at 1733770423032 (+20 ms)Flushing 1588230740/table: appending metadata at 1733770423044 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733770423044Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@636bbb14: reopening flushed file at 1733770423058 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12b77f13: reopening flushed file at 1733770423068 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4883f1d3: reopening flushed file at 1733770423076 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 115ms, sequenceid=18, compaction requested=false; wal=null at 1733770423084 (+8 ms)Cleaning up temporary data from old regions at 1733770423086 (+2 ms)Running coprocessor post-open hooks at 1733770423092 (+6 ms)Region opened successfully at 1733770423092 2024-12-09T18:53:43,093 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733770422884 2024-12-09T18:53:43,096 DEBUG [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T18:53:43,096 INFO [RS_OPEN_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T18:53:43,096 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:43,098 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a437f8b9ba7d,37501,1733770404084, state=OPEN 2024-12-09T18:53:43,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T18:53:43,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T18:53:43,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T18:53:43,137 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:43,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:43,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:43,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T18:53:43,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T18:53:43,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a437f8b9ba7d,37501,1733770404084 in 408 msec 2024-12-09T18:53:43,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-09T18:53:43,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 605 msec 2024-12-09T18:53:43,143 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): a437f8b9ba7d,46657,1733770404031 had 2 regions 2024-12-09T18:53:43,144 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a437f8b9ba7d,46657,1733770404031, splitWal=true, meta=true, isMeta: false 2024-12-09T18:53:43,146 INFO [PEWorker-1 {}] master.SplitWALManager(105): a437f8b9ba7d,46657,1733770404031 WAL count=1, meta=false 2024-12-09T18:53:43,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure a437f8b9ba7d%2C46657%2C1733770404031.1733770405845}] 2024-12-09T18:53:43,148 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:43,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure a437f8b9ba7d%2C46657%2C1733770404031.1733770405845, worker=a437f8b9ba7d,37501,1733770404084}] 2024-12-09T18:53:43,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37501 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-09T18:53:43,315 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845, size=2.2 K (2288bytes) 2024-12-09T18:53:43,315 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845 2024-12-09T18:53:43,316 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845 after 1ms 2024-12-09T18:53:43,339 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:43,339 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845 took 24ms 2024-12-09T18:53:43,342 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 58039aab64152816f1b072c38e494542: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-09T18:53:43,342 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845 so closing down 2024-12-09T18:53:43,342 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:43,343 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:43,343 INFO [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-09T18:53:43,343 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845, journal: Splitting hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845, size=2.2 K (2288bytes) at 1733770423315Finishing writing output for hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845 so closing down at 1733770423342 (+27 ms)3 split writer threads finished at 1733770423343 (+1 ms)Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733770423343 2024-12-09T18:53:43,343 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845 2024-12-09T18:53:43,343 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a437f8b9ba7d:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-09T18:53:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35375 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-09T18:53:43,347 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting/a437f8b9ba7d%2C46657%2C1733770404031.1733770405845 to hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/oldWALs 2024-12-09T18:53:43,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-09T18:53:43,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a437f8b9ba7d%2C46657%2C1733770404031.1733770405845, worker=a437f8b9ba7d,37501,1733770404084 in 198 msec 2024-12-09T18:53:43,351 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=a437f8b9ba7d,37501,1733770404084 2024-12-09T18:53:43,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-09T18:53:43,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure a437f8b9ba7d%2C46657%2C1733770404031.1733770405845, worker=a437f8b9ba7d,37501,1733770404084 in 205 msec 2024-12-09T18:53:43,355 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/WALs/a437f8b9ba7d,46657,1733770404031-splitting dir is empty, no logs to split. 2024-12-09T18:53:43,355 INFO [PEWorker-1 {}] master.SplitWALManager(105): a437f8b9ba7d,46657,1733770404031 WAL count=0, meta=false 2024-12-09T18:53:43,355 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if a437f8b9ba7d,46657,1733770404031 WAL splitting is done? wals=0, meta=false 2024-12-09T18:53:43,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, ASSIGN}] 2024-12-09T18:53:43,358 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, ASSIGN 2024-12-09T18:53:43,359 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T18:53:43,510 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(204): Hosts are {a437f8b9ba7d=0} racks are {/default-rack=0} 2024-12-09T18:53:43,510 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T18:53:43,510 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T18:53:43,510 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T18:53:43,510 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T18:53:43,510 INFO [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T18:53:43,510 INFO [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T18:53:43,510 DEBUG [a437f8b9ba7d:35375 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T18:53:43,511 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=OPENING, regionLocation=a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:43,512 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a437f8b9ba7d:46657 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a437f8b9ba7d/172.17.0.2:46657 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:43,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1, error=java.net.ConnectException: Call to address=a437f8b9ba7d:46657 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a437f8b9ba7d/172.17.0.2:46657 2024-12-09T18:53:43,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-09T18:53:43,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,46657,1733770404031, seqNum=-1 from cache 2024-12-09T18:53:43,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address a437f8b9ba7d:46657 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a437f8b9ba7d/172.17.0.2:46657 2024-12-09T18:53:43,529 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-09T18:53:43,530 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-09T18:53:43,531 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-09T18:53:43,531 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-09T18:53:43,615 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T18:53:43,615 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,37501,1733770404084, seqNum=-1] 2024-12-09T18:53:43,616 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T18:53:43,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52563, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T18:53:43,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, ASSIGN because future has completed 2024-12-09T18:53:43,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,35449,1733770403924}] 2024-12-09T18:53:43,776 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:43,776 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 58039aab64152816f1b072c38e494542, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:43,776 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,776 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:43,776 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,776 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,778 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,779 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf1 2024-12-09T18:53:43,779 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:43,785 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf1/e92e3827dea44bb9a5a9076577f710a4 2024-12-09T18:53:43,785 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:43,785 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,786 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58039aab64152816f1b072c38e494542 columnFamilyName cf2 2024-12-09T18:53:43,787 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:43,794 DEBUG [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/cf2/7697bbc23fb240f7a4d971818d3e69fa 2024-12-09T18:53:43,794 INFO [StoreOpener-58039aab64152816f1b072c38e494542-1 {}] regionserver.HStore(327): Store=58039aab64152816f1b072c38e494542/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:43,794 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,795 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,797 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,797 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,797 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,798 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T18:53:43,799 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,801 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 58039aab64152816f1b072c38e494542; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70764218, jitterRate=0.054469019174575806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T18:53:43,801 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58039aab64152816f1b072c38e494542 2024-12-09T18:53:43,802 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 58039aab64152816f1b072c38e494542: Running coprocessor pre-open hook at 1733770423777Writing region info on filesystem at 1733770423777Initializing all the Stores at 1733770423778 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770423778Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770423778Cleaning up temporary data from old regions at 1733770423797 (+19 ms)Running coprocessor post-open hooks at 1733770423801 (+4 ms)Region opened successfully at 1733770423802 (+1 ms) 2024-12-09T18:53:43,803 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., pid=21, masterSystemTime=1733770423772 2024-12-09T18:53:43,806 DEBUG [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:43,806 INFO [RS_OPEN_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:53:43,807 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=58039aab64152816f1b072c38e494542, regionState=OPEN, openSeqNum=18, regionLocation=a437f8b9ba7d,35449,1733770403924 2024-12-09T18:53:43,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,35449,1733770403924 because future has completed 2024-12-09T18:53:43,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-09T18:53:43,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 58039aab64152816f1b072c38e494542, server=a437f8b9ba7d,35449,1733770403924 in 191 msec 2024-12-09T18:53:43,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-09T18:53:43,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=58039aab64152816f1b072c38e494542, ASSIGN in 457 msec 2024-12-09T18:53:43,818 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server a437f8b9ba7d,46657,1733770404031 after splitting done 2024-12-09T18:53:43,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure a437f8b9ba7d,46657,1733770404031, splitWal=true, meta=true in 1.5720 sec 2024-12-09T18:53:43,969 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T18:53:44,025 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T18:53:44,027 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a437f8b9ba7d,37501,1733770404084, seqNum=-1] 2024-12-09T18:53:44,146 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to a437f8b9ba7d:46657 this server is in the failed servers list 2024-12-09T18:53:44,146 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,46657,1733770404031, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,46657,1733770404031, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=a437f8b9ba7d:46657 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a437f8b9ba7d:46657 2024-12-09T18:53:44,147 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,46657,1733770404031, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a437f8b9ba7d:46657 2024-12-09T18:53:44,147 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,46657,1733770404031, seqNum=12 from cache 2024-12-09T18:53:44,368 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542., hostname=a437f8b9ba7d,35449,1733770403924, seqNum=18] 2024-12-09T18:53:44,381 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 415), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=214 (was 189) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2827 (was 2890) 2024-12-09T18:53:44,382 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-09T18:53:44,395 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=410, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=214, ProcessCount=11, AvailableMemoryMB=2827 2024-12-09T18:53:44,395 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-09T18:53:44,408 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:44,409 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:44,410 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:44,413 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-86510351, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/hregion-86510351, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:44,424 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-86510351/hregion-86510351.1733770424413, exclude list is [], retry=0 2024-12-09T18:53:44,427 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:44,427 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:44,428 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:44,451 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-86510351/hregion-86510351.1733770424413 2024-12-09T18:53:44,452 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071)] 2024-12-09T18:53:44,452 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 50b2d5020d52d95630f6fcba302ccbd3, NAME => 'testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/hbase 2024-12-09T18:53:44,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741898_1076 (size=67) 2024-12-09T18:53:44,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741898_1076 (size=67) 2024-12-09T18:53:44,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741898_1076 (size=67) 2024-12-09T18:53:44,462 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:44,464 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,465 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50b2d5020d52d95630f6fcba302ccbd3 columnFamilyName a 2024-12-09T18:53:44,465 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,466 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(327): Store=50b2d5020d52d95630f6fcba302ccbd3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,466 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,467 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50b2d5020d52d95630f6fcba302ccbd3 columnFamilyName b 2024-12-09T18:53:44,467 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,468 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(327): Store=50b2d5020d52d95630f6fcba302ccbd3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,468 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,469 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50b2d5020d52d95630f6fcba302ccbd3 columnFamilyName c 2024-12-09T18:53:44,469 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,470 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(327): Store=50b2d5020d52d95630f6fcba302ccbd3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,470 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,470 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,471 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,472 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,472 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:44,473 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,475 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:44,476 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 50b2d5020d52d95630f6fcba302ccbd3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63796974, jitterRate=-0.049351006746292114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:44,476 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 50b2d5020d52d95630f6fcba302ccbd3: Writing region info on filesystem at 1733770424462Initializing all the Stores at 1733770424463 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424463Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424463Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424463Cleaning up temporary data from old regions at 1733770424472 (+9 ms)Region opened successfully at 1733770424476 (+4 ms) 2024-12-09T18:53:44,476 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 50b2d5020d52d95630f6fcba302ccbd3, disabling compactions & flushes 2024-12-09T18:53:44,476 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. 2024-12-09T18:53:44,476 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. 2024-12-09T18:53:44,476 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. after waiting 0 ms 2024-12-09T18:53:44,476 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. 2024-12-09T18:53:44,477 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. 2024-12-09T18:53:44,477 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 50b2d5020d52d95630f6fcba302ccbd3: Waiting for close lock at 1733770424476Disabling compacts and flushes for region at 1733770424476Disabling writes for close at 1733770424476Writing region close event to WAL at 1733770424477 (+1 ms)Closed at 1733770424477 2024-12-09T18:53:44,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741897_1075 (size=175) 2024-12-09T18:53:44,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741897_1075 (size=175) 2024-12-09T18:53:44,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741897_1075 (size=175) 2024-12-09T18:53:44,482 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:44,482 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-86510351:(num 1733770424413) 2024-12-09T18:53:44,482 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:44,483 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:44,495 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484, exclude list is [], retry=0 2024-12-09T18:53:44,497 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:44,498 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:44,498 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:44,520 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484 2024-12-09T18:53:44,520 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:36879:36879)] 2024-12-09T18:53:44,520 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 50b2d5020d52d95630f6fcba302ccbd3, NAME => 'testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:44,520 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:44,520 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,520 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,522 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,522 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50b2d5020d52d95630f6fcba302ccbd3 columnFamilyName a 2024-12-09T18:53:44,522 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,523 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(327): Store=50b2d5020d52d95630f6fcba302ccbd3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,523 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,524 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50b2d5020d52d95630f6fcba302ccbd3 columnFamilyName b 2024-12-09T18:53:44,524 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,524 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(327): Store=50b2d5020d52d95630f6fcba302ccbd3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,524 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,525 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50b2d5020d52d95630f6fcba302ccbd3 columnFamilyName c 2024-12-09T18:53:44,525 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,526 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(327): Store=50b2d5020d52d95630f6fcba302ccbd3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,526 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,526 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,527 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,528 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,528 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,529 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:44,530 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,530 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 50b2d5020d52d95630f6fcba302ccbd3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75457114, jitterRate=0.1243986189365387}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:44,531 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 50b2d5020d52d95630f6fcba302ccbd3: Writing region info on filesystem at 1733770424520Initializing all the Stores at 1733770424521 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424521Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424522 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424522Cleaning up temporary data from old regions at 1733770424528 (+6 ms)Region opened successfully at 1733770424531 (+3 ms) 2024-12-09T18:53:44,555 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 50b2d5020d52d95630f6fcba302ccbd3 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T18:53:44,574 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/a/7a48a15c1f1946cc9ed1290257ccf32e is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733770424531/Put/seqid=0 2024-12-09T18:53:44,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741900_1078 (size=5958) 2024-12-09T18:53:44,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741900_1078 (size=5958) 2024-12-09T18:53:44,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741900_1078 (size=5958) 2024-12-09T18:53:44,581 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/a/7a48a15c1f1946cc9ed1290257ccf32e 2024-12-09T18:53:44,606 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/b/85f52f8b8d60460eac66dcb3b649be3a is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733770424539/Put/seqid=0 2024-12-09T18:53:44,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741901_1079 (size=5958) 2024-12-09T18:53:44,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741901_1079 (size=5958) 2024-12-09T18:53:44,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741901_1079 (size=5958) 2024-12-09T18:53:44,612 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/b/85f52f8b8d60460eac66dcb3b649be3a 2024-12-09T18:53:44,631 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/c/f7889f178e1b4111ab4d3054365f2e81 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733770424546/Put/seqid=0 2024-12-09T18:53:44,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741902_1080 (size=5958) 2024-12-09T18:53:44,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741902_1080 (size=5958) 2024-12-09T18:53:44,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741902_1080 (size=5958) 2024-12-09T18:53:44,637 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/c/f7889f178e1b4111ab4d3054365f2e81 2024-12-09T18:53:44,643 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/a/7a48a15c1f1946cc9ed1290257ccf32e as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/a/7a48a15c1f1946cc9ed1290257ccf32e 2024-12-09T18:53:44,648 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/a/7a48a15c1f1946cc9ed1290257ccf32e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T18:53:44,649 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/b/85f52f8b8d60460eac66dcb3b649be3a as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/b/85f52f8b8d60460eac66dcb3b649be3a 2024-12-09T18:53:44,654 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/b/85f52f8b8d60460eac66dcb3b649be3a, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T18:53:44,655 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/c/f7889f178e1b4111ab4d3054365f2e81 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/c/f7889f178e1b4111ab4d3054365f2e81 2024-12-09T18:53:44,660 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/c/f7889f178e1b4111ab4d3054365f2e81, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T18:53:44,662 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 50b2d5020d52d95630f6fcba302ccbd3 in 106ms, sequenceid=33, compaction requested=false 2024-12-09T18:53:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 50b2d5020d52d95630f6fcba302ccbd3: 2024-12-09T18:53:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 50b2d5020d52d95630f6fcba302ccbd3, disabling compactions & flushes 2024-12-09T18:53:44,662 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. 2024-12-09T18:53:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. 2024-12-09T18:53:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. after waiting 0 ms 2024-12-09T18:53:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. 2024-12-09T18:53:44,663 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. 2024-12-09T18:53:44,663 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 50b2d5020d52d95630f6fcba302ccbd3: Waiting for close lock at 1733770424662Disabling compacts and flushes for region at 1733770424662Disabling writes for close at 1733770424662Writing region close event to WAL at 1733770424663 (+1 ms)Closed at 1733770424663 2024-12-09T18:53:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741899_1077 (size=6311) 2024-12-09T18:53:44,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741899_1077 (size=6311) 2024-12-09T18:53:44,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741899_1077 (size=6311) 2024-12-09T18:53:44,669 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/b/85f52f8b8d60460eac66dcb3b649be3a to hdfs://localhost:41577/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/b/85f52f8b8d60460eac66dcb3b649be3a 2024-12-09T18:53:44,682 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484, size=6.2 K (6311bytes) 2024-12-09T18:53:44,682 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484 2024-12-09T18:53:44,683 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484 after 1ms 2024-12-09T18:53:44,708 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:44,708 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484 took 26ms 2024-12-09T18:53:44,710 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484 so closing down 2024-12-09T18:53:44,710 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:44,711 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733770424484.temp 2024-12-09T18:53:44,735 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000003-wal.1733770424484.temp 2024-12-09T18:53:44,735 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:44,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741903_1081 (size=5575) 2024-12-09T18:53:44,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741903_1081 (size=5575) 2024-12-09T18:53:44,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741903_1081 (size=5575) 2024-12-09T18:53:44,741 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000003-wal.1733770424484.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T18:53:44,743 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000003-wal.1733770424484.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000032 2024-12-09T18:53:44,743 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-09T18:53:44,743 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484, journal: Splitting hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484, size=6.2 K (6311bytes) at 1733770424682Finishing writing output for hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484 so closing down at 1733770424710 (+28 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000003-wal.1733770424484.temp at 1733770424735 (+25 ms)3 split writer threads finished at 1733770424735Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000003-wal.1733770424484.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733770424741 (+6 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000003-wal.1733770424484.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000032 at 1733770424743 (+2 ms)Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733770424743 2024-12-09T18:53:44,744 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424484 to hdfs://localhost:41577/hbase/oldWALs/wal.1733770424484 2024-12-09T18:53:44,745 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000032 2024-12-09T18:53:44,745 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:44,747 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:44,758 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424747, exclude list is [], retry=0 2024-12-09T18:53:44,760 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:44,761 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:44,761 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:44,784 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424747 2024-12-09T18:53:44,784 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:36879:36879)] 2024-12-09T18:53:44,784 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 50b2d5020d52d95630f6fcba302ccbd3, NAME => 'testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:44,784 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:44,784 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,785 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,786 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,787 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50b2d5020d52d95630f6fcba302ccbd3 columnFamilyName a 2024-12-09T18:53:44,787 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,793 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/a/7a48a15c1f1946cc9ed1290257ccf32e 2024-12-09T18:53:44,793 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(327): Store=50b2d5020d52d95630f6fcba302ccbd3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,793 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,795 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50b2d5020d52d95630f6fcba302ccbd3 columnFamilyName b 2024-12-09T18:53:44,795 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,796 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(327): Store=50b2d5020d52d95630f6fcba302ccbd3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,796 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,797 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50b2d5020d52d95630f6fcba302ccbd3 columnFamilyName c 2024-12-09T18:53:44,797 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,802 DEBUG [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/c/f7889f178e1b4111ab4d3054365f2e81 2024-12-09T18:53:44,803 INFO [StoreOpener-50b2d5020d52d95630f6fcba302ccbd3-1 {}] regionserver.HStore(327): Store=50b2d5020d52d95630f6fcba302ccbd3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,803 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,805 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,806 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000032 2024-12-09T18:53:44,830 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:44,832 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000032 2024-12-09T18:53:44,832 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 50b2d5020d52d95630f6fcba302ccbd3 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T18:53:44,845 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/b/4d83b6efa32c49248621e98e44927a48 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733770424539/Put/seqid=0 2024-12-09T18:53:44,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741905_1083 (size=5958) 2024-12-09T18:53:44,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741905_1083 (size=5958) 2024-12-09T18:53:44,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741905_1083 (size=5958) 2024-12-09T18:53:44,852 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/b/4d83b6efa32c49248621e98e44927a48 2024-12-09T18:53:44,857 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/.tmp/b/4d83b6efa32c49248621e98e44927a48 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/b/4d83b6efa32c49248621e98e44927a48 2024-12-09T18:53:44,862 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/b/4d83b6efa32c49248621e98e44927a48, entries=10, sequenceid=32, filesize=5.8 K 2024-12-09T18:53:44,863 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 50b2d5020d52d95630f6fcba302ccbd3 in 30ms, sequenceid=32, compaction requested=false; wal=null 2024-12-09T18:53:44,863 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/0000000000000000032 2024-12-09T18:53:44,864 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,864 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,865 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:44,867 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 50b2d5020d52d95630f6fcba302ccbd3 2024-12-09T18:53:44,869 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/50b2d5020d52d95630f6fcba302ccbd3/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-09T18:53:44,871 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 50b2d5020d52d95630f6fcba302ccbd3; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65781753, jitterRate=-0.019775494933128357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:44,871 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 50b2d5020d52d95630f6fcba302ccbd3: Writing region info on filesystem at 1733770424785Initializing all the Stores at 1733770424786 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424786Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424786Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424786Obtaining lock to block concurrent updates at 1733770424832 (+46 ms)Preparing flush snapshotting stores in 50b2d5020d52d95630f6fcba302ccbd3 at 1733770424832Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733770424832Flushing stores of testReplayEditsWrittenViaHRegion,,1733770424408.50b2d5020d52d95630f6fcba302ccbd3. at 1733770424832Flushing 50b2d5020d52d95630f6fcba302ccbd3/b: creating writer at 1733770424832Flushing 50b2d5020d52d95630f6fcba302ccbd3/b: appending metadata at 1733770424844 (+12 ms)Flushing 50b2d5020d52d95630f6fcba302ccbd3/b: closing flushed file at 1733770424844Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@444715ce: reopening flushed file at 1733770424856 (+12 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 50b2d5020d52d95630f6fcba302ccbd3 in 30ms, sequenceid=32, compaction requested=false; wal=null at 1733770424863 (+7 ms)Cleaning up temporary data from old regions at 1733770424864 (+1 ms)Region opened successfully at 1733770424871 (+7 ms) 2024-12-09T18:53:44,890 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=420 (was 410) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:56736 [Waiting for operation #35] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:33168 [Waiting for operation #45] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:35516 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:47078 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:49780 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:43442 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=214 (was 214), ProcessCount=11 (was 11), AvailableMemoryMB=2818 (was 2827) 2024-12-09T18:53:44,890 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-09T18:53:44,901 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=214, ProcessCount=11, AvailableMemoryMB=2817 2024-12-09T18:53:44,901 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-09T18:53:44,914 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:44,916 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:44,917 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:44,919 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-07243135, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/hregion-07243135, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:44,932 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-07243135/hregion-07243135.1733770424920, exclude list is [], retry=0 2024-12-09T18:53:44,935 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:44,935 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:44,935 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:44,963 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-07243135/hregion-07243135.1733770424920 2024-12-09T18:53:44,963 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071)] 2024-12-09T18:53:44,964 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 41423e8e2b0d3ebfedd42d87d3fe31cf, NAME => 'testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/hbase 2024-12-09T18:53:44,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741907_1085 (size=68) 2024-12-09T18:53:44,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741907_1085 (size=68) 2024-12-09T18:53:44,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741907_1085 (size=68) 2024-12-09T18:53:44,974 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:44,977 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:44,979 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41423e8e2b0d3ebfedd42d87d3fe31cf columnFamilyName a 2024-12-09T18:53:44,979 DEBUG [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,980 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(327): Store=41423e8e2b0d3ebfedd42d87d3fe31cf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,980 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:44,981 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41423e8e2b0d3ebfedd42d87d3fe31cf columnFamilyName b 2024-12-09T18:53:44,982 DEBUG [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,982 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(327): Store=41423e8e2b0d3ebfedd42d87d3fe31cf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,982 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:44,984 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41423e8e2b0d3ebfedd42d87d3fe31cf columnFamilyName c 2024-12-09T18:53:44,984 DEBUG [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:44,984 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(327): Store=41423e8e2b0d3ebfedd42d87d3fe31cf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:44,985 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:44,985 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:44,986 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:44,987 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:44,987 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:44,987 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:44,988 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:44,990 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:44,991 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41423e8e2b0d3ebfedd42d87d3fe31cf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66495689, jitterRate=-0.009137019515037537}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:44,991 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41423e8e2b0d3ebfedd42d87d3fe31cf: Writing region info on filesystem at 1733770424974Initializing all the Stores at 1733770424975 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424975Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424976 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770424977 (+1 ms)Cleaning up temporary data from old regions at 1733770424987 (+10 ms)Region opened successfully at 1733770424991 (+4 ms) 2024-12-09T18:53:44,991 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 41423e8e2b0d3ebfedd42d87d3fe31cf, disabling compactions & flushes 2024-12-09T18:53:44,991 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. 2024-12-09T18:53:44,991 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. 2024-12-09T18:53:44,991 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. after waiting 0 ms 2024-12-09T18:53:44,991 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. 2024-12-09T18:53:44,992 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. 2024-12-09T18:53:44,992 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 41423e8e2b0d3ebfedd42d87d3fe31cf: Waiting for close lock at 1733770424991Disabling compacts and flushes for region at 1733770424991Disabling writes for close at 1733770424991Writing region close event to WAL at 1733770424992 (+1 ms)Closed at 1733770424992 2024-12-09T18:53:44,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741906_1084 (size=175) 2024-12-09T18:53:44,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741906_1084 (size=175) 2024-12-09T18:53:44,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741906_1084 (size=175) 2024-12-09T18:53:44,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:44,997 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-07243135:(num 1733770424920) 2024-12-09T18:53:44,997 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:45,000 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:45,012 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000, exclude list is [], retry=0 2024-12-09T18:53:45,015 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:45,015 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:45,016 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:45,042 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000 2024-12-09T18:53:45,042 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:45,101 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 41423e8e2b0d3ebfedd42d87d3fe31cf, NAME => 'testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:45,104 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,104 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:45,104 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,104 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,107 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,108 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41423e8e2b0d3ebfedd42d87d3fe31cf columnFamilyName a 2024-12-09T18:53:45,108 DEBUG [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:45,109 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(327): Store=41423e8e2b0d3ebfedd42d87d3fe31cf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:45,110 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,111 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41423e8e2b0d3ebfedd42d87d3fe31cf columnFamilyName b 2024-12-09T18:53:45,111 DEBUG [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:45,112 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(327): Store=41423e8e2b0d3ebfedd42d87d3fe31cf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:45,112 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,113 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41423e8e2b0d3ebfedd42d87d3fe31cf columnFamilyName c 2024-12-09T18:53:45,113 DEBUG [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:45,114 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(327): Store=41423e8e2b0d3ebfedd42d87d3fe31cf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:45,114 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,115 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,116 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,117 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,117 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,118 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:45,119 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,121 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41423e8e2b0d3ebfedd42d87d3fe31cf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66924519, jitterRate=-0.002746954560279846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:45,121 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:45,121 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41423e8e2b0d3ebfedd42d87d3fe31cf: Running coprocessor pre-open hook at 1733770425104Writing region info on filesystem at 1733770425104Initializing all the Stores at 1733770425106 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770425106Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770425106Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770425106Cleaning up temporary data from old regions at 1733770425117 (+11 ms)Running coprocessor post-open hooks at 1733770425121 (+4 ms)Region opened successfully at 1733770425121 2024-12-09T18:53:45,135 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-09T18:53:45,136 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 41423e8e2b0d3ebfedd42d87d3fe31cf/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:45,567 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T18:53:46,137 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 41423e8e2b0d3ebfedd42d87d3fe31cf/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:47,137 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 41423e8e2b0d3ebfedd42d87d3fe31cf/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:48,138 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 41423e8e2b0d3ebfedd42d87d3fe31cf/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:49,139 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 41423e8e2b0d3ebfedd42d87d3fe31cf/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:50,140 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 41423e8e2b0d3ebfedd42d87d3fe31cf/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:50,609 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T18:53:51,142 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 41423e8e2b0d3ebfedd42d87d3fe31cf/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:52,143 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 41423e8e2b0d3ebfedd42d87d3fe31cf/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:52,378 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T18:53:53,145 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 41423e8e2b0d3ebfedd42d87d3fe31cf/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:53,529 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-09T18:53:53,530 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-09T18:53:53,531 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T18:53:53,531 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-09T18:53:53,531 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-09T18:53:53,532 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-09T18:53:54,145 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 41423e8e2b0d3ebfedd42d87d3fe31cf/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:54,146 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 41423e8e2b0d3ebfedd42d87d3fe31cf: 2024-12-09T18:53:54,146 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. 2024-12-09T18:53:54,158 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 41423e8e2b0d3ebfedd42d87d3fe31cf: 2024-12-09T18:53:54,158 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-09T18:53:54,158 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 41423e8e2b0d3ebfedd42d87d3fe31cf, disabling compactions & flushes 2024-12-09T18:53:54,158 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. 2024-12-09T18:53:54,158 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. 2024-12-09T18:53:54,158 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. after waiting 0 ms 2024-12-09T18:53:54,158 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. 2024-12-09T18:53:54,159 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. 2024-12-09T18:53:54,159 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. 2024-12-09T18:53:54,159 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 41423e8e2b0d3ebfedd42d87d3fe31cf: Waiting for close lock at 1733770434158Running coprocessor pre-close hooks at 1733770434158Disabling compacts and flushes for region at 1733770434158Disabling writes for close at 1733770434158Writing region close event to WAL at 1733770434159 (+1 ms)Running coprocessor post-close hooks at 1733770434159Closed at 1733770434159 2024-12-09T18:53:54,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741908_1086 (size=4236) 2024-12-09T18:53:54,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741908_1086 (size=4236) 2024-12-09T18:53:54,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741908_1086 (size=4236) 2024-12-09T18:53:54,176 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000, size=4.1 K (4236bytes) 2024-12-09T18:53:54,176 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000 2024-12-09T18:53:54,177 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000 after 1ms 2024-12-09T18:53:54,200 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:54,200 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000 took 24ms 2024-12-09T18:53:54,202 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000 so closing down 2024-12-09T18:53:54,202 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:54,204 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733770425000.temp 2024-12-09T18:53:54,227 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000004-wal.1733770425000.temp 2024-12-09T18:53:54,227 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:54,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741909_1087 (size=3245) 2024-12-09T18:53:54,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741909_1087 (size=3245) 2024-12-09T18:53:54,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741909_1087 (size=3245) 2024-12-09T18:53:54,234 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000004-wal.1733770425000.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-09T18:53:54,235 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000004-wal.1733770425000.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000026 2024-12-09T18:53:54,235 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-09T18:53:54,235 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000, journal: Splitting hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000, size=4.1 K (4236bytes) at 1733770434176Finishing writing output for hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000 so closing down at 1733770434202 (+26 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000004-wal.1733770425000.temp at 1733770434227 (+25 ms)3 split writer threads finished at 1733770434227Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000004-wal.1733770425000.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733770434234 (+7 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000004-wal.1733770425000.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000026 at 1733770434235 (+1 ms)Processed 23 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733770434235 2024-12-09T18:53:54,237 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770425000 to hdfs://localhost:41577/hbase/oldWALs/wal.1733770425000 2024-12-09T18:53:54,238 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000026 2024-12-09T18:53:54,238 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:54,239 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:54,253 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770434239, exclude list is [], retry=0 2024-12-09T18:53:54,255 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:54,256 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:54,256 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:54,280 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770434239 2024-12-09T18:53:54,281 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:54,281 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 41423e8e2b0d3ebfedd42d87d3fe31cf, NAME => 'testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:54,281 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,281 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:54,282 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,282 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,283 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,284 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41423e8e2b0d3ebfedd42d87d3fe31cf columnFamilyName a 2024-12-09T18:53:54,284 DEBUG [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:54,285 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(327): Store=41423e8e2b0d3ebfedd42d87d3fe31cf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:54,285 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,285 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41423e8e2b0d3ebfedd42d87d3fe31cf columnFamilyName b 2024-12-09T18:53:54,286 DEBUG [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:54,286 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(327): Store=41423e8e2b0d3ebfedd42d87d3fe31cf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:54,286 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,287 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41423e8e2b0d3ebfedd42d87d3fe31cf columnFamilyName c 2024-12-09T18:53:54,287 DEBUG [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:54,287 INFO [StoreOpener-41423e8e2b0d3ebfedd42d87d3fe31cf-1 {}] regionserver.HStore(327): Store=41423e8e2b0d3ebfedd42d87d3fe31cf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:54,287 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,288 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,289 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,289 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000026 2024-12-09T18:53:54,312 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:54,313 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000026 2024-12-09T18:53:54,313 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-09T18:53:54,326 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/.tmp/a/d219d303360f421a8d0c44ed54fcd007 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733770434150/Put/seqid=0 2024-12-09T18:53:54,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741911_1089 (size=5523) 2024-12-09T18:53:54,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741911_1089 (size=5523) 2024-12-09T18:53:54,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741911_1089 (size=5523) 2024-12-09T18:53:54,333 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/.tmp/a/d219d303360f421a8d0c44ed54fcd007 2024-12-09T18:53:54,360 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/.tmp/b/dbd237f3700b4b4e9bea402999629150 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733770434147/Put/seqid=0 2024-12-09T18:53:54,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741912_1090 (size=5524) 2024-12-09T18:53:54,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741912_1090 (size=5524) 2024-12-09T18:53:54,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741912_1090 (size=5524) 2024-12-09T18:53:54,368 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/.tmp/b/dbd237f3700b4b4e9bea402999629150 2024-12-09T18:53:54,418 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/.tmp/c/0d1032b0e05d4c0aa381f23020f9de79 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733770434148/Put/seqid=0 2024-12-09T18:53:54,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741913_1091 (size=5457) 2024-12-09T18:53:54,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741913_1091 (size=5457) 2024-12-09T18:53:54,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741913_1091 (size=5457) 2024-12-09T18:53:54,428 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/.tmp/c/0d1032b0e05d4c0aa381f23020f9de79 2024-12-09T18:53:54,435 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/.tmp/a/d219d303360f421a8d0c44ed54fcd007 as hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/a/d219d303360f421a8d0c44ed54fcd007 2024-12-09T18:53:54,443 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/a/d219d303360f421a8d0c44ed54fcd007, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T18:53:54,444 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/.tmp/b/dbd237f3700b4b4e9bea402999629150 as hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/b/dbd237f3700b4b4e9bea402999629150 2024-12-09T18:53:54,451 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/b/dbd237f3700b4b4e9bea402999629150, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T18:53:54,453 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/.tmp/c/0d1032b0e05d4c0aa381f23020f9de79 as hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/c/0d1032b0e05d4c0aa381f23020f9de79 2024-12-09T18:53:54,458 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/c/0d1032b0e05d4c0aa381f23020f9de79, entries=6, sequenceid=26, filesize=5.3 K 2024-12-09T18:53:54,459 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 41423e8e2b0d3ebfedd42d87d3fe31cf in 145ms, sequenceid=26, compaction requested=false; wal=null 2024-12-09T18:53:54,486 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/0000000000000000026 2024-12-09T18:53:54,487 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,487 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,488 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:54,490 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,492 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsAfterAbortingFlush/41423e8e2b0d3ebfedd42d87d3fe31cf/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-09T18:53:54,493 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41423e8e2b0d3ebfedd42d87d3fe31cf; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74214202, jitterRate=0.10587778687477112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:54,493 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 41423e8e2b0d3ebfedd42d87d3fe31cf 2024-12-09T18:53:54,493 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41423e8e2b0d3ebfedd42d87d3fe31cf: Running coprocessor pre-open hook at 1733770434282Writing region info on filesystem at 1733770434282Initializing all the Stores at 1733770434282Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770434283 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770434283Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770434283Obtaining lock to block concurrent updates at 1733770434313 (+30 ms)Preparing flush snapshotting stores in 41423e8e2b0d3ebfedd42d87d3fe31cf at 1733770434313Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733770434313Flushing stores of testReplayEditsAfterAbortingFlush,,1733770424915.41423e8e2b0d3ebfedd42d87d3fe31cf. at 1733770434313Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf/a: creating writer at 1733770434313Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf/a: appending metadata at 1733770434325 (+12 ms)Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf/a: closing flushed file at 1733770434326 (+1 ms)Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf/b: creating writer at 1733770434339 (+13 ms)Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf/b: appending metadata at 1733770434359 (+20 ms)Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf/b: closing flushed file at 1733770434360 (+1 ms)Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf/c: creating writer at 1733770434374 (+14 ms)Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf/c: appending metadata at 1733770434417 (+43 ms)Flushing 41423e8e2b0d3ebfedd42d87d3fe31cf/c: closing flushed file at 1733770434417Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3255ced4: reopening flushed file at 1733770434434 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c30393d: reopening flushed file at 1733770434443 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d3ea2cd: reopening flushed file at 1733770434452 (+9 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 41423e8e2b0d3ebfedd42d87d3fe31cf in 145ms, sequenceid=26, compaction requested=false; wal=null at 1733770434459 (+7 ms)Cleaning up temporary data from old regions at 1733770434487 (+28 ms)Running coprocessor post-open hooks at 1733770434493 (+6 ms)Region opened successfully at 1733770434493 2024-12-09T18:53:54,513 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420 (was 420), OpenFileDescriptor=1180 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=189 (was 214), ProcessCount=11 (was 11), AvailableMemoryMB=3180 (was 2817) - AvailableMemoryMB LEAK? - 2024-12-09T18:53:54,513 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-09T18:53:54,524 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=420, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=189, ProcessCount=11, AvailableMemoryMB=3179 2024-12-09T18:53:54,524 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-09T18:53:54,539 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:54,541 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:54,541 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:54,543 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-83212228, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/hregion-83212228, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:54,557 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-83212228/hregion-83212228.1733770434544, exclude list is [], retry=0 2024-12-09T18:53:54,560 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:54,560 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:54,561 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:54,588 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-83212228/hregion-83212228.1733770434544 2024-12-09T18:53:54,588 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:54,588 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e5d760492e64d64b4c6df94408b6d744, NAME => 'testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/hbase 2024-12-09T18:53:54,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741915_1093 (size=61) 2024-12-09T18:53:54,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741915_1093 (size=61) 2024-12-09T18:53:54,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741915_1093 (size=61) 2024-12-09T18:53:54,602 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:54,603 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,605 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5d760492e64d64b4c6df94408b6d744 columnFamilyName a 2024-12-09T18:53:54,605 DEBUG [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:54,605 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] regionserver.HStore(327): Store=e5d760492e64d64b4c6df94408b6d744/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:54,605 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,606 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,607 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,607 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,607 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,608 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,611 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:54,611 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5d760492e64d64b4c6df94408b6d744; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63556378, jitterRate=-0.052936166524887085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T18:53:54,612 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5d760492e64d64b4c6df94408b6d744: Writing region info on filesystem at 1733770434602Initializing all the Stores at 1733770434603 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770434603Cleaning up temporary data from old regions at 1733770434607 (+4 ms)Region opened successfully at 1733770434612 (+5 ms) 2024-12-09T18:53:54,612 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e5d760492e64d64b4c6df94408b6d744, disabling compactions & flushes 2024-12-09T18:53:54,612 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. 2024-12-09T18:53:54,612 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. 2024-12-09T18:53:54,612 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. after waiting 0 ms 2024-12-09T18:53:54,612 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. 2024-12-09T18:53:54,612 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. 2024-12-09T18:53:54,612 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e5d760492e64d64b4c6df94408b6d744: Waiting for close lock at 1733770434612Disabling compacts and flushes for region at 1733770434612Disabling writes for close at 1733770434612Writing region close event to WAL at 1733770434612Closed at 1733770434612 2024-12-09T18:53:54,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741914_1092 (size=175) 2024-12-09T18:53:54,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741914_1092 (size=175) 2024-12-09T18:53:54,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741914_1092 (size=175) 2024-12-09T18:53:54,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:54,620 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-83212228:(num 1733770434544) 2024-12-09T18:53:54,620 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:54,622 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:54,633 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622, exclude list is [], retry=0 2024-12-09T18:53:54,636 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:54,636 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:54,636 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:54,660 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622 2024-12-09T18:53:54,660 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879)] 2024-12-09T18:53:54,660 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e5d760492e64d64b4c6df94408b6d744, NAME => 'testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:54,660 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:54,660 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,660 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,662 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,663 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5d760492e64d64b4c6df94408b6d744 columnFamilyName a 2024-12-09T18:53:54,663 DEBUG [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:54,663 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] regionserver.HStore(327): Store=e5d760492e64d64b4c6df94408b6d744/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:54,663 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,664 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,665 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,666 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,666 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,668 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:54,669 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5d760492e64d64b4c6df94408b6d744; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59985528, jitterRate=-0.10614597797393799}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T18:53:54,669 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5d760492e64d64b4c6df94408b6d744: Writing region info on filesystem at 1733770434660Initializing all the Stores at 1733770434661 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770434661Cleaning up temporary data from old regions at 1733770434666 (+5 ms)Region opened successfully at 1733770434669 (+3 ms) 2024-12-09T18:53:54,678 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e5d760492e64d64b4c6df94408b6d744, disabling compactions & flushes 2024-12-09T18:53:54,678 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. 2024-12-09T18:53:54,678 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. 2024-12-09T18:53:54,678 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. after waiting 0 ms 2024-12-09T18:53:54,678 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. 2024-12-09T18:53:54,678 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. 2024-12-09T18:53:54,679 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. 2024-12-09T18:53:54,679 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e5d760492e64d64b4c6df94408b6d744: Waiting for close lock at 1733770434678Disabling compacts and flushes for region at 1733770434678Disabling writes for close at 1733770434678Writing region close event to WAL at 1733770434678Closed at 1733770434678 2024-12-09T18:53:54,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741916_1094 (size=1795) 2024-12-09T18:53:54,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741916_1094 (size=1795) 2024-12-09T18:53:54,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741916_1094 (size=1795) 2024-12-09T18:53:54,698 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622, size=1.8 K (1795bytes) 2024-12-09T18:53:54,698 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622 2024-12-09T18:53:54,699 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622 after 1ms 2024-12-09T18:53:54,723 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:54,723 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622 took 25ms 2024-12-09T18:53:54,725 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622 so closing down 2024-12-09T18:53:54,725 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:53:54,727 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733770434622.temp 2024-12-09T18:53:54,749 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000003-wal.1733770434622.temp 2024-12-09T18:53:54,750 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:53:54,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741917_1095 (size=1795) 2024-12-09T18:53:54,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741917_1095 (size=1795) 2024-12-09T18:53:54,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741917_1095 (size=1795) 2024-12-09T18:53:54,767 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000003-wal.1733770434622.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-09T18:53:54,768 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000003-wal.1733770434622.temp to hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000012 2024-12-09T18:53:54,769 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-09T18:53:54,769 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622, journal: Splitting hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622, size=1.8 K (1795bytes) at 1733770434698Finishing writing output for hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622 so closing down at 1733770434725 (+27 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000003-wal.1733770434622.temp at 1733770434749 (+24 ms)3 split writer threads finished at 1733770434750 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000003-wal.1733770434622.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733770434767 (+17 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000003-wal.1733770434622.temp to hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000012 at 1733770434768 (+1 ms)Processed 10 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733770434769 (+1 ms) 2024-12-09T18:53:54,770 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770434622 to hdfs://localhost:41577/hbase/oldWALs/wal.1733770434622 2024-12-09T18:53:54,771 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000012 2024-12-09T18:53:54,799 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:55,217 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:55,219 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:55,230 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770435219, exclude list is [], retry=0 2024-12-09T18:53:55,232 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:55,232 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:55,232 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:55,255 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770435219 2024-12-09T18:53:55,255 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879)] 2024-12-09T18:53:55,255 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e5d760492e64d64b4c6df94408b6d744, NAME => 'testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:55,255 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:55,255 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,255 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,258 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,260 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5d760492e64d64b4c6df94408b6d744 columnFamilyName a 2024-12-09T18:53:55,260 DEBUG [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:55,261 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] regionserver.HStore(327): Store=e5d760492e64d64b4c6df94408b6d744/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:55,261 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,261 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,262 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,263 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000012 2024-12-09T18:53:55,288 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:53:55,289 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000012 2024-12-09T18:53:55,289 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e5d760492e64d64b4c6df94408b6d744 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-09T18:53:55,306 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/.tmp/a/4f3f2dcf74a4489b9710444ec876c56a is 79, key is testDatalossWhenInputError/a:x0/1733770434669/Put/seqid=0 2024-12-09T18:53:55,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741919_1097 (size=5808) 2024-12-09T18:53:55,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741919_1097 (size=5808) 2024-12-09T18:53:55,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741919_1097 (size=5808) 2024-12-09T18:53:55,312 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/.tmp/a/4f3f2dcf74a4489b9710444ec876c56a 2024-12-09T18:53:55,321 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/.tmp/a/4f3f2dcf74a4489b9710444ec876c56a as hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/a/4f3f2dcf74a4489b9710444ec876c56a 2024-12-09T18:53:55,331 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/a/4f3f2dcf74a4489b9710444ec876c56a, entries=10, sequenceid=12, filesize=5.7 K 2024-12-09T18:53:55,331 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for e5d760492e64d64b4c6df94408b6d744 in 42ms, sequenceid=12, compaction requested=false; wal=null 2024-12-09T18:53:55,332 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/0000000000000000012 2024-12-09T18:53:55,333 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,333 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,336 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,338 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-09T18:53:55,339 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5d760492e64d64b4c6df94408b6d744; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68123873, jitterRate=0.015124812722206116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T18:53:55,340 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5d760492e64d64b4c6df94408b6d744: Writing region info on filesystem at 1733770435256Initializing all the Stores at 1733770435258 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770435258Obtaining lock to block concurrent updates at 1733770435289 (+31 ms)Preparing flush snapshotting stores in e5d760492e64d64b4c6df94408b6d744 at 1733770435289Finished memstore snapshotting testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733770435289Flushing stores of testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744. at 1733770435289Flushing e5d760492e64d64b4c6df94408b6d744/a: creating writer at 1733770435289Flushing e5d760492e64d64b4c6df94408b6d744/a: appending metadata at 1733770435305 (+16 ms)Flushing e5d760492e64d64b4c6df94408b6d744/a: closing flushed file at 1733770435305Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20024ef3: reopening flushed file at 1733770435320 (+15 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for e5d760492e64d64b4c6df94408b6d744 in 42ms, sequenceid=12, compaction requested=false; wal=null at 1733770435331 (+11 ms)Cleaning up temporary data from old regions at 1733770435333 (+2 ms)Region opened successfully at 1733770435339 (+6 ms) 2024-12-09T18:53:55,343 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e5d760492e64d64b4c6df94408b6d744, NAME => 'testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:55,343 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733770434539.e5d760492e64d64b4c6df94408b6d744.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:55,343 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,343 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,345 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,345 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5d760492e64d64b4c6df94408b6d744 columnFamilyName a 2024-12-09T18:53:55,345 DEBUG [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:55,351 DEBUG [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/a/4f3f2dcf74a4489b9710444ec876c56a 2024-12-09T18:53:55,351 INFO [StoreOpener-e5d760492e64d64b4c6df94408b6d744-1 {}] regionserver.HStore(327): Store=e5d760492e64d64b4c6df94408b6d744/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:55,351 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,352 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,353 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,354 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,354 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,355 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5d760492e64d64b4c6df94408b6d744 2024-12-09T18:53:55,357 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testDatalossWhenInputError/e5d760492e64d64b4c6df94408b6d744/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-09T18:53:55,358 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5d760492e64d64b4c6df94408b6d744; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71036880, jitterRate=0.058531999588012695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T18:53:55,358 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5d760492e64d64b4c6df94408b6d744: Writing region info on filesystem at 1733770435343Initializing all the Stores at 1733770435344 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770435344Cleaning up temporary data from old regions at 1733770435354 (+10 ms)Region opened successfully at 1733770435358 (+4 ms) 2024-12-09T18:53:55,376 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=431 (was 420) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:41064 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:54270 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:54174 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:46206 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:46238 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:41000 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1882736132-172.17.0.2-1733770399228:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=189 (was 189), ProcessCount=11 (was 11), AvailableMemoryMB=3096 (was 3179) 2024-12-09T18:53:55,376 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-09T18:53:55,389 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=431, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=189, ProcessCount=11, AvailableMemoryMB=3094 2024-12-09T18:53:55,389 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-09T18:53:55,409 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:55,411 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:53:55,412 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:53:55,415 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-53171400, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/hregion-53171400, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:55,427 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-53171400/hregion-53171400.1733770435415, exclude list is [], retry=0 2024-12-09T18:53:55,430 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:55,430 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:55,431 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:55,473 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-53171400/hregion-53171400.1733770435415 2024-12-09T18:53:55,475 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:53:55,475 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8a150664d724c333878e8a6e11b5b607, NAME => 'testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/hbase 2024-12-09T18:53:55,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741921_1099 (size=63) 2024-12-09T18:53:55,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741921_1099 (size=63) 2024-12-09T18:53:55,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741921_1099 (size=63) 2024-12-09T18:53:55,488 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:55,491 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,493 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a150664d724c333878e8a6e11b5b607 columnFamilyName a 2024-12-09T18:53:55,493 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:55,494 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(327): Store=8a150664d724c333878e8a6e11b5b607/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:55,494 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,496 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a150664d724c333878e8a6e11b5b607 columnFamilyName b 2024-12-09T18:53:55,496 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:55,496 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(327): Store=8a150664d724c333878e8a6e11b5b607/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:55,497 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,498 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a150664d724c333878e8a6e11b5b607 columnFamilyName c 2024-12-09T18:53:55,498 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:55,499 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(327): Store=8a150664d724c333878e8a6e11b5b607/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:55,499 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,500 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,500 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,504 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,504 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,505 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:55,506 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,508 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:53:55,508 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8a150664d724c333878e8a6e11b5b607; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65414472, jitterRate=-0.025248408317565918}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:55,509 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8a150664d724c333878e8a6e11b5b607: Writing region info on filesystem at 1733770435489Initializing all the Stores at 1733770435489Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770435489Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770435491 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770435491Cleaning up temporary data from old regions at 1733770435504 (+13 ms)Region opened successfully at 1733770435509 (+5 ms) 2024-12-09T18:53:55,509 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8a150664d724c333878e8a6e11b5b607, disabling compactions & flushes 2024-12-09T18:53:55,509 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. 2024-12-09T18:53:55,509 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. 2024-12-09T18:53:55,509 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. after waiting 0 ms 2024-12-09T18:53:55,509 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. 2024-12-09T18:53:55,509 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. 2024-12-09T18:53:55,509 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8a150664d724c333878e8a6e11b5b607: Waiting for close lock at 1733770435509Disabling compacts and flushes for region at 1733770435509Disabling writes for close at 1733770435509Writing region close event to WAL at 1733770435509Closed at 1733770435509 2024-12-09T18:53:55,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741920_1098 (size=175) 2024-12-09T18:53:55,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741920_1098 (size=175) 2024-12-09T18:53:55,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741920_1098 (size=175) 2024-12-09T18:53:55,516 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:53:55,516 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-53171400:(num 1733770435415) 2024-12-09T18:53:55,516 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:53:55,518 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:53:55,536 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518, exclude list is [], retry=0 2024-12-09T18:53:55,539 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:53:55,540 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:53:55,540 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:53:55,580 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 2024-12-09T18:53:55,580 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879)] 2024-12-09T18:53:55,581 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a150664d724c333878e8a6e11b5b607, NAME => 'testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:53:55,581 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:53:55,581 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,581 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,583 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,584 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a150664d724c333878e8a6e11b5b607 columnFamilyName a 2024-12-09T18:53:55,584 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:55,584 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(327): Store=8a150664d724c333878e8a6e11b5b607/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:55,585 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,586 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a150664d724c333878e8a6e11b5b607 columnFamilyName b 2024-12-09T18:53:55,586 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:55,586 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(327): Store=8a150664d724c333878e8a6e11b5b607/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:55,586 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,587 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a150664d724c333878e8a6e11b5b607 columnFamilyName c 2024-12-09T18:53:55,588 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:53:55,588 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(327): Store=8a150664d724c333878e8a6e11b5b607/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:53:55,588 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,589 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,591 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,592 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,592 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,592 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:53:55,595 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:53:55,596 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8a150664d724c333878e8a6e11b5b607; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63959401, jitterRate=-0.04693065583705902}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:53:55,597 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8a150664d724c333878e8a6e11b5b607: Writing region info on filesystem at 1733770435581Initializing all the Stores at 1733770435582 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770435582Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770435583 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770435583Cleaning up temporary data from old regions at 1733770435592 (+9 ms)Region opened successfully at 1733770435597 (+5 ms) 2024-12-09T18:53:55,603 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733770435602/Put/seqid=0 2024-12-09T18:53:55,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741923_1101 (size=4875) 2024-12-09T18:53:55,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741923_1101 (size=4875) 2024-12-09T18:53:55,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741923_1101 (size=4875) 2024-12-09T18:53:55,625 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733770435624/Put/seqid=0 2024-12-09T18:53:55,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741924_1102 (size=4875) 2024-12-09T18:53:55,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741924_1102 (size=4875) 2024-12-09T18:53:55,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741924_1102 (size=4875) 2024-12-09T18:53:56,043 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733770436043/Put/seqid=0 2024-12-09T18:53:56,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741925_1103 (size=4875) 2024-12-09T18:53:56,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741925_1103 (size=4875) 2024-12-09T18:53:56,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741925_1103 (size=4875) 2024-12-09T18:53:56,054 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 8a150664d724c333878e8a6e11b5b607/a 2024-12-09T18:53:56,057 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-09T18:53:56,057 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T18:53:56,057 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 8a150664d724c333878e8a6e11b5b607/a 2024-12-09T18:53:56,060 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-09T18:53:56,060 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T18:53:56,060 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 8a150664d724c333878e8a6e11b5b607/a 2024-12-09T18:53:56,063 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-09T18:53:56,064 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T18:53:56,064 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8a150664d724c333878e8a6e11b5b607 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-09T18:53:56,084 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/.tmp/a/05e4e95b860344b7a69857629bb454b5 is 55, key is testCompactedBulkLoadedFiles/a:a/1733770435597/Put/seqid=0 2024-12-09T18:53:56,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741926_1104 (size=5107) 2024-12-09T18:53:56,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741926_1104 (size=5107) 2024-12-09T18:53:56,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741926_1104 (size=5107) 2024-12-09T18:53:56,092 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/.tmp/a/05e4e95b860344b7a69857629bb454b5 2024-12-09T18:53:56,099 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/.tmp/a/05e4e95b860344b7a69857629bb454b5 as hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/05e4e95b860344b7a69857629bb454b5 2024-12-09T18:53:56,105 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/05e4e95b860344b7a69857629bb454b5, entries=1, sequenceid=4, filesize=5.0 K 2024-12-09T18:53:56,106 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 8a150664d724c333878e8a6e11b5b607 in 42ms, sequenceid=4, compaction requested=false 2024-12-09T18:53:56,107 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8a150664d724c333878e8a6e11b5b607: 2024-12-09T18:53:56,108 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/76576fcc190d4b37b3c8860f976bca05_SeqId_4_ 2024-12-09T18:53:56,109 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_ 2024-12-09T18:53:56,110 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_ 2024-12-09T18:53:56,110 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile0 into 8a150664d724c333878e8a6e11b5b607/a as hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/76576fcc190d4b37b3c8860f976bca05_SeqId_4_ - updating store file list. 2024-12-09T18:53:56,115 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 76576fcc190d4b37b3c8860f976bca05_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T18:53:56,115 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/76576fcc190d4b37b3c8860f976bca05_SeqId_4_ into 8a150664d724c333878e8a6e11b5b607/a 2024-12-09T18:53:56,115 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile0 into 8a150664d724c333878e8a6e11b5b607/a (new location: hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/76576fcc190d4b37b3c8860f976bca05_SeqId_4_) 2024-12-09T18:53:56,117 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile1 into 8a150664d724c333878e8a6e11b5b607/a as hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_ - updating store file list. 2024-12-09T18:53:56,122 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T18:53:56,122 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_ into 8a150664d724c333878e8a6e11b5b607/a 2024-12-09T18:53:56,122 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile1 into 8a150664d724c333878e8a6e11b5b607/a (new location: hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_) 2024-12-09T18:53:56,123 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile2 into 8a150664d724c333878e8a6e11b5b607/a as hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_ - updating store file list. 2024-12-09T18:53:56,129 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T18:53:56,129 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_ into 8a150664d724c333878e8a6e11b5b607/a 2024-12-09T18:53:56,129 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:41577/hbase/testCompactedBulkLoadedFiles/hfile2 into 8a150664d724c333878e8a6e11b5b607/a (new location: hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_) 2024-12-09T18:53:56,137 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T18:53:56,138 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8a150664d724c333878e8a6e11b5b607/a is initiating major compaction (all files) 2024-12-09T18:53:56,138 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8a150664d724c333878e8a6e11b5b607/a in testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. 2024-12-09T18:53:56,138 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/05e4e95b860344b7a69857629bb454b5, hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/76576fcc190d4b37b3c8860f976bca05_SeqId_4_, hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_, hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_] into tmpdir=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/.tmp, totalSize=19.3 K 2024-12-09T18:53:56,138 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 05e4e95b860344b7a69857629bb454b5, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733770435597 2024-12-09T18:53:56,139 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 76576fcc190d4b37b3c8860f976bca05_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T18:53:56,139 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T18:53:56,140 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T18:53:56,153 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/.tmp/a/e3ea79c334bb4de59552a7c36954c3c7 is 55, key is testCompactedBulkLoadedFiles/a:a/1733770435597/Put/seqid=0 2024-12-09T18:53:56,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741927_1105 (size=6154) 2024-12-09T18:53:56,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741927_1105 (size=6154) 2024-12-09T18:53:56,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741927_1105 (size=6154) 2024-12-09T18:53:56,169 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/.tmp/a/e3ea79c334bb4de59552a7c36954c3c7 as hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e3ea79c334bb4de59552a7c36954c3c7 2024-12-09T18:53:56,174 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 8a150664d724c333878e8a6e11b5b607/a of 8a150664d724c333878e8a6e11b5b607 into e3ea79c334bb4de59552a7c36954c3c7(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T18:53:56,174 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8a150664d724c333878e8a6e11b5b607: 2024-12-09T18:53:56,174 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T18:53:56,175 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T18:53:56,201 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518, size=0 (0bytes) 2024-12-09T18:53:56,201 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 might be still open, length is 0 2024-12-09T18:53:56,201 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 2024-12-09T18:53:56,202 WARN [IPC Server handler 0 on default port 41577 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-09T18:53:56,202 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 after 1ms 2024-12-09T18:53:58,303 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:46262 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:36059:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46262 dst: /127.0.0.1:36059 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36059 remote=/127.0.0.1:46262]. Total timeout mills is 60000, 57871 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:58,303 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:41110 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38029:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41110 dst: /127.0.0.1:38029 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:58,303 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:54316 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37407:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54316 dst: /127.0.0.1:37407 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:53:58,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741922_1106 (size=1794) 2024-12-09T18:53:58,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741922_1106 (size=1794) 2024-12-09T18:53:58,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741922_1106 (size=1794) 2024-12-09T18:53:59,661 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T18:54:00,203 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 after 4002ms 2024-12-09T18:54:00,229 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:54:00,229 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 took 4028ms 2024-12-09T18:54:00,231 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518; continuing. 2024-12-09T18:54:00,231 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 so closing down 2024-12-09T18:54:00,231 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:54:00,233 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733770435518.temp 2024-12-09T18:54:00,270 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000003-wal.1733770435518.temp 2024-12-09T18:54:00,271 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:54:00,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741928_1107 (size=800) 2024-12-09T18:54:00,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741928_1107 (size=800) 2024-12-09T18:54:00,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741928_1107 (size=800) 2024-12-09T18:54:00,281 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000003-wal.1733770435518.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T18:54:00,283 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000003-wal.1733770435518.temp to hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000008 2024-12-09T18:54:00,283 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 54 ms; skipped=3; WAL=hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T18:54:00,283 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518, journal: Splitting hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518, size=0 (0bytes) at 1733770436201Finishing writing output for hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 so closing down at 1733770440231 (+4030 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000003-wal.1733770435518.temp at 1733770440270 (+39 ms)3 split writer threads finished at 1733770440271 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000003-wal.1733770435518.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733770440281 (+10 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000003-wal.1733770435518.temp to hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000008 at 1733770440283 (+2 ms)Processed 5 edits across 1 Regions in 54 ms; skipped=3; WAL=hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518, size=0, length=0, corrupted=false, cancelled=false at 1733770440283 2024-12-09T18:54:00,285 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 to hdfs://localhost:41577/hbase/oldWALs/wal.1733770435518 2024-12-09T18:54:00,286 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000008 2024-12-09T18:54:00,286 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:54:00,288 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:54:00,302 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770440288, exclude list is [], retry=0 2024-12-09T18:54:00,304 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:54:00,305 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:54:00,305 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:54:00,331 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770440288 2024-12-09T18:54:00,332 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:54:00,332 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a150664d724c333878e8a6e11b5b607, NAME => 'testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:54:00,332 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:54:00,332 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,332 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,334 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,334 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a150664d724c333878e8a6e11b5b607 columnFamilyName a 2024-12-09T18:54:00,334 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,340 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/05e4e95b860344b7a69857629bb454b5 2024-12-09T18:54:00,344 DEBUG [StoreFileOpener-8a150664d724c333878e8a6e11b5b607-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T18:54:00,344 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_ 2024-12-09T18:54:00,348 DEBUG [StoreFileOpener-8a150664d724c333878e8a6e11b5b607-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 76576fcc190d4b37b3c8860f976bca05_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T18:54:00,348 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/76576fcc190d4b37b3c8860f976bca05_SeqId_4_ 2024-12-09T18:54:00,351 DEBUG [StoreFileOpener-8a150664d724c333878e8a6e11b5b607-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T18:54:00,352 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_ 2024-12-09T18:54:00,355 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e3ea79c334bb4de59552a7c36954c3c7 2024-12-09T18:54:00,355 WARN [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/05e4e95b860344b7a69857629bb454b5 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@782fc757 2024-12-09T18:54:00,355 WARN [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@782fc757 2024-12-09T18:54:00,355 WARN [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/76576fcc190d4b37b3c8860f976bca05_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@782fc757 2024-12-09T18:54:00,356 WARN [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@782fc757 2024-12-09T18:54:00,356 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/05e4e95b860344b7a69857629bb454b5, hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_, hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/76576fcc190d4b37b3c8860f976bca05_SeqId_4_, hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_] to archive 2024-12-09T18:54:00,357 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T18:54:00,359 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/05e4e95b860344b7a69857629bb454b5 to hdfs://localhost:41577/hbase/archive/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/05e4e95b860344b7a69857629bb454b5 2024-12-09T18:54:00,360 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_ to hdfs://localhost:41577/hbase/archive/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_ 2024-12-09T18:54:00,361 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/76576fcc190d4b37b3c8860f976bca05_SeqId_4_ to hdfs://localhost:41577/hbase/archive/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/76576fcc190d4b37b3c8860f976bca05_SeqId_4_ 2024-12-09T18:54:00,362 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_ to hdfs://localhost:41577/hbase/archive/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/a/e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_ 2024-12-09T18:54:00,362 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(327): Store=8a150664d724c333878e8a6e11b5b607/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,362 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,363 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a150664d724c333878e8a6e11b5b607 columnFamilyName b 2024-12-09T18:54:00,363 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,363 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(327): Store=8a150664d724c333878e8a6e11b5b607/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,364 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,364 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a150664d724c333878e8a6e11b5b607 columnFamilyName c 2024-12-09T18:54:00,364 DEBUG [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,365 INFO [StoreOpener-8a150664d724c333878e8a6e11b5b607-1 {}] regionserver.HStore(327): Store=8a150664d724c333878e8a6e11b5b607/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,365 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,365 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,367 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,367 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000008 2024-12-09T18:54:00,389 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:54:00,391 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 8a150664d724c333878e8a6e11b5b607 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "8a150664d724c333878e8a6e11b5b607" family_name: "a" compaction_input: "05e4e95b860344b7a69857629bb454b5" compaction_input: "76576fcc190d4b37b3c8860f976bca05_SeqId_4_" compaction_input: "3efeed0b966c45f6b5a66c2fc373127e_SeqId_4_" compaction_input: "e2e73f8efcc748d2930ce89dba5f835f_SeqId_4_" compaction_output: "e3ea79c334bb4de59552a7c36954c3c7" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-09T18:54:00,391 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-09T18:54:00,391 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000008 2024-12-09T18:54:00,392 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/0000000000000000008 2024-12-09T18:54:00,393 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,393 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,393 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:54:00,395 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8a150664d724c333878e8a6e11b5b607 2024-12-09T18:54:00,397 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testCompactedBulkLoadedFiles/8a150664d724c333878e8a6e11b5b607/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T18:54:00,398 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8a150664d724c333878e8a6e11b5b607; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62785575, jitterRate=-0.06442202627658844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:54:00,398 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8a150664d724c333878e8a6e11b5b607: Writing region info on filesystem at 1733770440332Initializing all the Stores at 1733770440333 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440333Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440333Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440333Cleaning up temporary data from old regions at 1733770440393 (+60 ms)Region opened successfully at 1733770440398 (+5 ms) 2024-12-09T18:54:00,400 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8a150664d724c333878e8a6e11b5b607, disabling compactions & flushes 2024-12-09T18:54:00,400 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. 2024-12-09T18:54:00,400 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. 2024-12-09T18:54:00,400 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. after waiting 0 ms 2024-12-09T18:54:00,400 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. 2024-12-09T18:54:00,401 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733770435409.8a150664d724c333878e8a6e11b5b607. 2024-12-09T18:54:00,401 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8a150664d724c333878e8a6e11b5b607: Waiting for close lock at 1733770440400Disabling compacts and flushes for region at 1733770440400Disabling writes for close at 1733770440400Writing region close event to WAL at 1733770440401 (+1 ms)Closed at 1733770440401 2024-12-09T18:54:00,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741929_1108 (size=175) 2024-12-09T18:54:00,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741929_1108 (size=175) 2024-12-09T18:54:00,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741929_1108 (size=175) 2024-12-09T18:54:00,405 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:54:00,405 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733770440288) 2024-12-09T18:54:00,417 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=444 (was 431) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1916322176_22 at /127.0.0.1:42754 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:41577 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1916322176_22 at /127.0.0.1:49942 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1916322176_22 at /127.0.0.1:53224 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:41577 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 189) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2709 (was 3094) 2024-12-09T18:54:00,417 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-09T18:54:00,429 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=2708 2024-12-09T18:54:00,429 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-09T18:54:00,442 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:54:00,443 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T18:54:00,444 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T18:54:00,446 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-43053076, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/hregion-43053076, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:54:00,459 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-43053076/hregion-43053076.1733770440447, exclude list is [], retry=0 2024-12-09T18:54:00,462 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:54:00,463 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:54:00,464 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:54:00,490 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-43053076/hregion-43053076.1733770440447 2024-12-09T18:54:00,491 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879)] 2024-12-09T18:54:00,491 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9243fedc839a4999e93f3b223ae31a65, NAME => 'testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41577/hbase 2024-12-09T18:54:00,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741931_1110 (size=67) 2024-12-09T18:54:00,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741931_1110 (size=67) 2024-12-09T18:54:00,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741931_1110 (size=67) 2024-12-09T18:54:00,499 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:54:00,500 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,501 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName a 2024-12-09T18:54:00,501 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,502 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,502 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,503 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName b 2024-12-09T18:54:00,503 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,503 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,503 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,504 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName c 2024-12-09T18:54:00,504 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,505 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,505 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,505 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,505 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,506 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,506 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,506 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:54:00,507 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,509 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T18:54:00,509 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9243fedc839a4999e93f3b223ae31a65; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74942369, jitterRate=0.11672832071781158}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:54:00,509 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9243fedc839a4999e93f3b223ae31a65: Writing region info on filesystem at 1733770440499Initializing all the Stores at 1733770440500 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440500Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440500Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440500Cleaning up temporary data from old regions at 1733770440506 (+6 ms)Region opened successfully at 1733770440509 (+3 ms) 2024-12-09T18:54:00,510 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9243fedc839a4999e93f3b223ae31a65, disabling compactions & flushes 2024-12-09T18:54:00,510 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:00,510 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:00,510 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. after waiting 0 ms 2024-12-09T18:54:00,510 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:00,510 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:00,510 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9243fedc839a4999e93f3b223ae31a65: Waiting for close lock at 1733770440510Disabling compacts and flushes for region at 1733770440510Disabling writes for close at 1733770440510Writing region close event to WAL at 1733770440510Closed at 1733770440510 2024-12-09T18:54:00,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741930_1109 (size=175) 2024-12-09T18:54:00,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741930_1109 (size=175) 2024-12-09T18:54:00,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741930_1109 (size=175) 2024-12-09T18:54:00,514 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:54:00,514 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-43053076:(num 1733770440447) 2024-12-09T18:54:00,514 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:54:00,516 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:54:00,531 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516, exclude list is [], retry=0 2024-12-09T18:54:00,534 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:54:00,534 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:54:00,534 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:54:00,559 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516 2024-12-09T18:54:00,559 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:54:00,560 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9243fedc839a4999e93f3b223ae31a65, NAME => 'testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:54:00,560 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:54:00,560 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,560 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,561 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,562 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName a 2024-12-09T18:54:00,562 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,563 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,563 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,563 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName b 2024-12-09T18:54:00,563 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,564 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,564 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,565 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName c 2024-12-09T18:54:00,565 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,565 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,565 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,566 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,567 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,567 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,567 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,568 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:54:00,569 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,570 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9243fedc839a4999e93f3b223ae31a65; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68402219, jitterRate=0.019272491335868835}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:54:00,570 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9243fedc839a4999e93f3b223ae31a65: Writing region info on filesystem at 1733770440560Initializing all the Stores at 1733770440561 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440561Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440561Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440561Cleaning up temporary data from old regions at 1733770440567 (+6 ms)Region opened successfully at 1733770440570 (+3 ms) 2024-12-09T18:54:00,577 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9243fedc839a4999e93f3b223ae31a65 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T18:54:00,590 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/a/455b87e0cd6f4c8e99bc707c60b39271 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733770440570/Put/seqid=0 2024-12-09T18:54:00,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741933_1112 (size=5958) 2024-12-09T18:54:00,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741933_1112 (size=5958) 2024-12-09T18:54:00,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741933_1112 (size=5958) 2024-12-09T18:54:00,596 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/a/455b87e0cd6f4c8e99bc707c60b39271 2024-12-09T18:54:00,601 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/a/455b87e0cd6f4c8e99bc707c60b39271 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/a/455b87e0cd6f4c8e99bc707c60b39271 2024-12-09T18:54:00,606 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/a/455b87e0cd6f4c8e99bc707c60b39271, entries=10, sequenceid=13, filesize=5.8 K 2024-12-09T18:54:00,607 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 9243fedc839a4999e93f3b223ae31a65 in 29ms, sequenceid=13, compaction requested=false 2024-12-09T18:54:00,607 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9243fedc839a4999e93f3b223ae31a65: 2024-12-09T18:54:00,622 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9243fedc839a4999e93f3b223ae31a65, disabling compactions & flushes 2024-12-09T18:54:00,622 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:00,622 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:00,622 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. after waiting 0 ms 2024-12-09T18:54:00,622 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:00,623 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:00,623 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:00,623 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9243fedc839a4999e93f3b223ae31a65: Waiting for close lock at 1733770440622Disabling compacts and flushes for region at 1733770440622Disabling writes for close at 1733770440622Writing region close event to WAL at 1733770440623 (+1 ms)Closed at 1733770440623 2024-12-09T18:54:00,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741932_1111 (size=6243) 2024-12-09T18:54:00,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741932_1111 (size=6243) 2024-12-09T18:54:00,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741932_1111 (size=6243) 2024-12-09T18:54:00,641 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516, size=6.1 K (6243bytes) 2024-12-09T18:54:00,641 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516 2024-12-09T18:54:00,641 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516 after 0ms 2024-12-09T18:54:00,662 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:54:00,663 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516 took 22ms 2024-12-09T18:54:00,664 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516 so closing down 2024-12-09T18:54:00,665 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:54:00,666 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733770440516.temp 2024-12-09T18:54:00,690 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000003-wal.1733770440516.temp 2024-12-09T18:54:00,690 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:54:00,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741934_1113 (size=5575) 2024-12-09T18:54:00,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741934_1113 (size=5575) 2024-12-09T18:54:00,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741934_1113 (size=5575) 2024-12-09T18:54:00,696 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000003-wal.1733770440516.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T18:54:00,697 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000003-wal.1733770440516.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000035 2024-12-09T18:54:00,697 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-09T18:54:00,698 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516, journal: Splitting hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516, size=6.1 K (6243bytes) at 1733770440641Finishing writing output for hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516 so closing down at 1733770440664 (+23 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000003-wal.1733770440516.temp at 1733770440690 (+26 ms)3 split writer threads finished at 1733770440690Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000003-wal.1733770440516.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733770440696 (+6 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000003-wal.1733770440516.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000035 at 1733770440697 (+1 ms)Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733770440697 2024-12-09T18:54:00,699 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440516 to hdfs://localhost:41577/hbase/oldWALs/wal.1733770440516 2024-12-09T18:54:00,700 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000035 2024-12-09T18:54:00,700 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:54:00,701 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:54:00,713 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701, exclude list is [], retry=0 2024-12-09T18:54:00,715 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:54:00,716 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:54:00,716 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:54:00,753 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 2024-12-09T18:54:00,755 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:44937:44937),(127.0.0.1/127.0.0.1:40071:40071)] 2024-12-09T18:54:00,755 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9243fedc839a4999e93f3b223ae31a65, NAME => 'testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65.', STARTKEY => '', ENDKEY => ''} 2024-12-09T18:54:00,756 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:54:00,756 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,756 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,757 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,758 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName a 2024-12-09T18:54:00,758 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,764 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/a/455b87e0cd6f4c8e99bc707c60b39271 2024-12-09T18:54:00,764 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,764 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,765 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName b 2024-12-09T18:54:00,765 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,766 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,766 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,767 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName c 2024-12-09T18:54:00,767 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:00,768 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:00,768 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,769 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,770 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:00,771 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000035 2024-12-09T18:54:00,801 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:54:00,802 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000035 2024-12-09T18:54:00,803 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9243fedc839a4999e93f3b223ae31a65 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-09T18:54:00,823 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/b/8809bf7980744966800e5d9bf9c501f3 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733770440607/Put/seqid=0 2024-12-09T18:54:00,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741936_1115 (size=5958) 2024-12-09T18:54:00,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741936_1115 (size=5958) 2024-12-09T18:54:00,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741936_1115 (size=5958) 2024-12-09T18:54:01,233 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/b/8809bf7980744966800e5d9bf9c501f3 2024-12-09T18:54:01,252 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/c/455db3dda11c4727ac4d0aee0f338b69 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733770440612/Put/seqid=0 2024-12-09T18:54:01,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741937_1116 (size=5958) 2024-12-09T18:54:01,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741937_1116 (size=5958) 2024-12-09T18:54:01,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741937_1116 (size=5958) 2024-12-09T18:54:01,263 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/c/455db3dda11c4727ac4d0aee0f338b69 2024-12-09T18:54:01,270 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/b/8809bf7980744966800e5d9bf9c501f3 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/b/8809bf7980744966800e5d9bf9c501f3 2024-12-09T18:54:01,277 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/b/8809bf7980744966800e5d9bf9c501f3, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T18:54:01,278 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/c/455db3dda11c4727ac4d0aee0f338b69 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/c/455db3dda11c4727ac4d0aee0f338b69 2024-12-09T18:54:01,283 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/c/455db3dda11c4727ac4d0aee0f338b69, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T18:54:01,283 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 9243fedc839a4999e93f3b223ae31a65 in 481ms, sequenceid=35, compaction requested=false; wal=null 2024-12-09T18:54:01,284 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000035 2024-12-09T18:54:01,285 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:01,285 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:01,286 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:54:01,288 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:01,292 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-09T18:54:01,293 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9243fedc839a4999e93f3b223ae31a65; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65175538, jitterRate=-0.028808802366256714}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:54:01,293 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9243fedc839a4999e93f3b223ae31a65: Writing region info on filesystem at 1733770440756Initializing all the Stores at 1733770440757 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440757Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440757Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770440757Obtaining lock to block concurrent updates at 1733770440803 (+46 ms)Preparing flush snapshotting stores in 9243fedc839a4999e93f3b223ae31a65 at 1733770440803Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733770440803Flushing stores of testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. at 1733770440803Flushing 9243fedc839a4999e93f3b223ae31a65/b: creating writer at 1733770440803Flushing 9243fedc839a4999e93f3b223ae31a65/b: appending metadata at 1733770440822 (+19 ms)Flushing 9243fedc839a4999e93f3b223ae31a65/b: closing flushed file at 1733770440822Flushing 9243fedc839a4999e93f3b223ae31a65/c: creating writer at 1733770441238 (+416 ms)Flushing 9243fedc839a4999e93f3b223ae31a65/c: appending metadata at 1733770441252 (+14 ms)Flushing 9243fedc839a4999e93f3b223ae31a65/c: closing flushed file at 1733770441252Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53c8122d: reopening flushed file at 1733770441269 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42e76534: reopening flushed file at 1733770441277 (+8 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 9243fedc839a4999e93f3b223ae31a65 in 481ms, sequenceid=35, compaction requested=false; wal=null at 1733770441283 (+6 ms)Cleaning up temporary data from old regions at 1733770441285 (+2 ms)Region opened successfully at 1733770441293 (+8 ms) 2024-12-09T18:54:01,371 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701, size=0 (0bytes) 2024-12-09T18:54:01,371 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 might be still open, length is 0 2024-12-09T18:54:01,371 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 2024-12-09T18:54:01,372 WARN [IPC Server handler 0 on default port 41577 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-09T18:54:01,372 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 after 1ms 2024-12-09T18:54:03,529 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-09T18:54:03,530 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-09T18:54:03,531 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-09T18:54:03,531 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-09T18:54:04,302 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:53276 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38029:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53276 dst: /127.0.0.1:38029 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38029 remote=/127.0.0.1:53276]. Total timeout mills is 60000, 57022 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:54:04,303 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:50018 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:36059:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50018 dst: /127.0.0.1:36059 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:54:04,303 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-339328517_22 at /127.0.0.1:42802 [Receiving block BP-1882736132-172.17.0.2-1733770399228:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37407:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42802 dst: /127.0.0.1:37407 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:54:04,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741935_1117 (size=5567) 2024-12-09T18:54:04,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741935_1117 (size=5567) 2024-12-09T18:54:05,374 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 after 4003ms 2024-12-09T18:54:05,451 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:54:05,451 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 took 4080ms 2024-12-09T18:54:05,454 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701; continuing. 2024-12-09T18:54:05,454 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 so closing down 2024-12-09T18:54:05,454 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T18:54:05,455 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733770440701.temp 2024-12-09T18:54:05,480 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000037-wal.1733770440701.temp 2024-12-09T18:54:05,480 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T18:54:05,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741938_1118 (size=5575) 2024-12-09T18:54:05,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741938_1118 (size=5575) 2024-12-09T18:54:05,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741938_1118 (size=5575) 2024-12-09T18:54:05,487 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000037-wal.1733770440701.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T18:54:05,489 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000037-wal.1733770440701.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000066 2024-12-09T18:54:05,489 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T18:54:05,489 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701, journal: Splitting hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701, size=0 (0bytes) at 1733770441371Finishing writing output for hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 so closing down at 1733770445454 (+4083 ms)Creating recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000037-wal.1733770440701.temp at 1733770445480 (+26 ms)3 split writer threads finished at 1733770445481 (+1 ms)Closed recovered edits writer path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000037-wal.1733770440701.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733770445487 (+6 ms)Rename recovered edits hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000037-wal.1733770440701.temp to hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000066 at 1733770445489 (+2 ms)Processed 30 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701, size=0, length=0, corrupted=false, cancelled=false at 1733770445489 2024-12-09T18:54:05,490 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 to hdfs://localhost:41577/hbase/oldWALs/wal.1733770440701 2024-12-09T18:54:05,491 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000066 2024-12-09T18:54:05,491 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T18:54:05,493 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:41577/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441, archiveDir=hdfs://localhost:41577/hbase/oldWALs, maxLogs=32 2024-12-09T18:54:05,504 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770445493, exclude list is [], retry=0 2024-12-09T18:54:05,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38029,DS-78736f6e-11e0-47ba-a24a-c9d2194cefa6,DISK] 2024-12-09T18:54:05,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37407,DS-09c47033-c5b3-479d-880f-5fa06bff2e3a,DISK] 2024-12-09T18:54:05,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36059,DS-f51ad991-2b9c-4d92-ad1b-0d05754b24f6,DISK] 2024-12-09T18:54:05,530 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770445493 2024-12-09T18:54:05,530 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36879:36879),(127.0.0.1/127.0.0.1:40071:40071),(127.0.0.1/127.0.0.1:44937:44937)] 2024-12-09T18:54:05,530 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T18:54:05,532 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:05,532 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName a 2024-12-09T18:54:05,532 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:05,537 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/a/455b87e0cd6f4c8e99bc707c60b39271 2024-12-09T18:54:05,537 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:05,537 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:05,538 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName b 2024-12-09T18:54:05,538 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:05,543 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/b/8809bf7980744966800e5d9bf9c501f3 2024-12-09T18:54:05,543 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:05,544 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:05,544 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9243fedc839a4999e93f3b223ae31a65 columnFamilyName c 2024-12-09T18:54:05,545 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T18:54:05,550 DEBUG [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/c/455db3dda11c4727ac4d0aee0f338b69 2024-12-09T18:54:05,550 INFO [StoreOpener-9243fedc839a4999e93f3b223ae31a65-1 {}] regionserver.HStore(327): Store=9243fedc839a4999e93f3b223ae31a65/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T18:54:05,550 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:05,551 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:05,553 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:05,553 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000066 2024-12-09T18:54:05,578 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T18:54:05,582 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000066 2024-12-09T18:54:05,582 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9243fedc839a4999e93f3b223ae31a65 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T18:54:05,596 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/a/bbab7693ef184660945a2ed200e15d7a is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733770441299/Put/seqid=0 2024-12-09T18:54:05,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741940_1120 (size=5958) 2024-12-09T18:54:05,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741940_1120 (size=5958) 2024-12-09T18:54:05,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741940_1120 (size=5958) 2024-12-09T18:54:05,603 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/a/bbab7693ef184660945a2ed200e15d7a 2024-12-09T18:54:05,624 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/b/1e5c63a7a96443ee8f06a138d7feb045 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733770441309/Put/seqid=0 2024-12-09T18:54:05,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741941_1121 (size=5958) 2024-12-09T18:54:05,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741941_1121 (size=5958) 2024-12-09T18:54:05,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741941_1121 (size=5958) 2024-12-09T18:54:05,631 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/b/1e5c63a7a96443ee8f06a138d7feb045 2024-12-09T18:54:05,648 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/c/188e5a7ad4ce4f9e9ef608620ce1d1c8 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733770441317/Put/seqid=0 2024-12-09T18:54:05,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741942_1122 (size=5958) 2024-12-09T18:54:05,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741942_1122 (size=5958) 2024-12-09T18:54:05,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741942_1122 (size=5958) 2024-12-09T18:54:05,654 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/c/188e5a7ad4ce4f9e9ef608620ce1d1c8 2024-12-09T18:54:05,659 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/a/bbab7693ef184660945a2ed200e15d7a as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/a/bbab7693ef184660945a2ed200e15d7a 2024-12-09T18:54:05,663 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/a/bbab7693ef184660945a2ed200e15d7a, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T18:54:05,664 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/b/1e5c63a7a96443ee8f06a138d7feb045 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/b/1e5c63a7a96443ee8f06a138d7feb045 2024-12-09T18:54:05,668 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/b/1e5c63a7a96443ee8f06a138d7feb045, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T18:54:05,669 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/.tmp/c/188e5a7ad4ce4f9e9ef608620ce1d1c8 as hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/c/188e5a7ad4ce4f9e9ef608620ce1d1c8 2024-12-09T18:54:05,674 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/c/188e5a7ad4ce4f9e9ef608620ce1d1c8, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T18:54:05,674 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 9243fedc839a4999e93f3b223ae31a65 in 92ms, sequenceid=66, compaction requested=false; wal=null 2024-12-09T18:54:05,675 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/0000000000000000066 2024-12-09T18:54:05,676 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:05,676 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:05,677 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T18:54:05,678 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9243fedc839a4999e93f3b223ae31a65 2024-12-09T18:54:05,680 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/hbase/data/default/testReplayEditsWrittenViaHRegion/9243fedc839a4999e93f3b223ae31a65/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-09T18:54:05,681 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9243fedc839a4999e93f3b223ae31a65; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67863396, jitterRate=0.01124340295791626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T18:54:05,681 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9243fedc839a4999e93f3b223ae31a65: Writing region info on filesystem at 1733770445530Initializing all the Stores at 1733770445531 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770445531Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770445531Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733770445531Obtaining lock to block concurrent updates at 1733770445582 (+51 ms)Preparing flush snapshotting stores in 9243fedc839a4999e93f3b223ae31a65 at 1733770445582Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733770445582Flushing stores of testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. at 1733770445582Flushing 9243fedc839a4999e93f3b223ae31a65/a: creating writer at 1733770445583 (+1 ms)Flushing 9243fedc839a4999e93f3b223ae31a65/a: appending metadata at 1733770445596 (+13 ms)Flushing 9243fedc839a4999e93f3b223ae31a65/a: closing flushed file at 1733770445596Flushing 9243fedc839a4999e93f3b223ae31a65/b: creating writer at 1733770445607 (+11 ms)Flushing 9243fedc839a4999e93f3b223ae31a65/b: appending metadata at 1733770445624 (+17 ms)Flushing 9243fedc839a4999e93f3b223ae31a65/b: closing flushed file at 1733770445624Flushing 9243fedc839a4999e93f3b223ae31a65/c: creating writer at 1733770445636 (+12 ms)Flushing 9243fedc839a4999e93f3b223ae31a65/c: appending metadata at 1733770445647 (+11 ms)Flushing 9243fedc839a4999e93f3b223ae31a65/c: closing flushed file at 1733770445647Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@258f7104: reopening flushed file at 1733770445658 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43f40768: reopening flushed file at 1733770445663 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f1a8753: reopening flushed file at 1733770445669 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 9243fedc839a4999e93f3b223ae31a65 in 92ms, sequenceid=66, compaction requested=false; wal=null at 1733770445674 (+5 ms)Cleaning up temporary data from old regions at 1733770445676 (+2 ms)Region opened successfully at 1733770445681 (+5 ms) 2024-12-09T18:54:05,692 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9243fedc839a4999e93f3b223ae31a65, disabling compactions & flushes 2024-12-09T18:54:05,692 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:05,692 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:05,692 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. after waiting 0 ms 2024-12-09T18:54:05,692 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:05,694 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733770440442.9243fedc839a4999e93f3b223ae31a65. 2024-12-09T18:54:05,694 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9243fedc839a4999e93f3b223ae31a65: Waiting for close lock at 1733770445692Disabling compacts and flushes for region at 1733770445692Disabling writes for close at 1733770445692Writing region close event to WAL at 1733770445694 (+2 ms)Closed at 1733770445694 2024-12-09T18:54:05,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741939_1119 (size=175) 2024-12-09T18:54:05,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741939_1119 (size=175) 2024-12-09T18:54:05,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741939_1119 (size=175) 2024-12-09T18:54:05,698 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T18:54:05,698 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733770445493) 2024-12-09T18:54:05,710 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=449 (was 444) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1653292493_22 at /127.0.0.1:53306 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:41577 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:41577 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1653292493_22 at /127.0.0.1:50050 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1653292493_22 at /127.0.0.1:42836 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=204 (was 222), ProcessCount=11 (was 11), AvailableMemoryMB=2648 (was 2708) 2024-12-09T18:54:05,711 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-09T18:54:05,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T18:54:05,711 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T18:54:05,711 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T18:54:05,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T18:54:05,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T18:54:05,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T18:54:05,712 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T18:54:05,712 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1062718288, stopped=false 2024-12-09T18:54:05,712 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a437f8b9ba7d,35375,1733770403171 2024-12-09T18:54:05,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T18:54:05,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T18:54:05,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T18:54:05,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:54:05,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:54:05,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:54:05,725 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T18:54:05,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T18:54:05,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T18:54:05,726 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T18:54:05,727 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T18:54:05,727 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T18:54:05,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T18:54:05,727 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a437f8b9ba7d,35449,1733770403924' ***** 2024-12-09T18:54:05,727 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T18:54:05,727 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a437f8b9ba7d,37501,1733770404084' ***** 2024-12-09T18:54:05,727 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T18:54:05,728 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T18:54:05,728 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T18:54:05,728 INFO [RS:2;a437f8b9ba7d:37501 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T18:54:05,728 INFO [RS:0;a437f8b9ba7d:35449 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T18:54:05,728 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T18:54:05,728 INFO [RS:2;a437f8b9ba7d:37501 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T18:54:05,728 INFO [RS:0;a437f8b9ba7d:35449 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T18:54:05,728 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T18:54:05,728 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(959): stopping server a437f8b9ba7d,37501,1733770404084 2024-12-09T18:54:05,728 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(3091): Received CLOSE for 58039aab64152816f1b072c38e494542 2024-12-09T18:54:05,728 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T18:54:05,728 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(959): stopping server a437f8b9ba7d,35449,1733770403924 2024-12-09T18:54:05,729 INFO [RS:2;a437f8b9ba7d:37501 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a437f8b9ba7d:37501. 2024-12-09T18:54:05,729 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T18:54:05,729 INFO [RS:0;a437f8b9ba7d:35449 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a437f8b9ba7d:35449. 2024-12-09T18:54:05,729 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 58039aab64152816f1b072c38e494542, disabling compactions & flushes 2024-12-09T18:54:05,729 DEBUG [RS:2;a437f8b9ba7d:37501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T18:54:05,729 DEBUG [RS:0;a437f8b9ba7d:35449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T18:54:05,729 DEBUG [RS:2;a437f8b9ba7d:37501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T18:54:05,729 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:54:05,729 DEBUG [RS:0;a437f8b9ba7d:35449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T18:54:05,729 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:54:05,729 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T18:54:05,729 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T18:54:05,729 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. after waiting 0 ms 2024-12-09T18:54:05,729 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T18:54:05,729 DEBUG [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(1325): Online Regions={58039aab64152816f1b072c38e494542=testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542.} 2024-12-09T18:54:05,729 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:54:05,729 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T18:54:05,729 DEBUG [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(1351): Waiting on 58039aab64152816f1b072c38e494542 2024-12-09T18:54:05,729 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T18:54:05,729 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T18:54:05,730 DEBUG [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T18:54:05,730 DEBUG [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T18:54:05,730 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T18:54:05,730 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T18:54:05,730 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T18:54:05,730 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T18:54:05,730 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T18:54:05,730 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-09T18:54:05,735 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/58039aab64152816f1b072c38e494542/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-09T18:54:05,736 INFO [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:54:05,736 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 58039aab64152816f1b072c38e494542: Waiting for close lock at 1733770445728Running coprocessor pre-close hooks at 1733770445728Disabling compacts and flushes for region at 1733770445728Disabling writes for close at 1733770445729 (+1 ms)Writing region close event to WAL at 1733770445730 (+1 ms)Running coprocessor post-close hooks at 1733770445736 (+6 ms)Closed at 1733770445736 2024-12-09T18:54:05,736 DEBUG [RS_CLOSE_REGION-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542. 2024-12-09T18:54:05,736 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/info/5c55bca932e14910add6b72725975631 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733770419634.58039aab64152816f1b072c38e494542./info:regioninfo/1733770423806/Put/seqid=0 2024-12-09T18:54:05,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741943_1123 (size=6778) 2024-12-09T18:54:05,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741943_1123 (size=6778) 2024-12-09T18:54:05,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741943_1123 (size=6778) 2024-12-09T18:54:05,742 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/info/5c55bca932e14910add6b72725975631 2024-12-09T18:54:05,746 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/.tmp/info/5c55bca932e14910add6b72725975631 as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/info/5c55bca932e14910add6b72725975631 2024-12-09T18:54:05,751 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/info/5c55bca932e14910add6b72725975631, entries=8, sequenceid=23, filesize=6.6 K 2024-12-09T18:54:05,752 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 21ms, sequenceid=23, compaction requested=false 2024-12-09T18:54:05,756 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-09T18:54:05,757 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T18:54:05,757 INFO [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T18:54:05,757 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733770445730Running coprocessor pre-close hooks at 1733770445730Disabling compacts and flushes for region at 1733770445730Disabling writes for close at 1733770445730Obtaining lock to block concurrent updates at 1733770445730Preparing flush snapshotting stores in 1588230740 at 1733770445730Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733770445731 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733770445731Flushing 1588230740/info: creating writer at 1733770445732 (+1 ms)Flushing 1588230740/info: appending metadata at 1733770445736 (+4 ms)Flushing 1588230740/info: closing flushed file at 1733770445736Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@268ad068: reopening flushed file at 1733770445745 (+9 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 21ms, sequenceid=23, compaction requested=false at 1733770445752 (+7 ms)Writing region close event to WAL at 1733770445753 (+1 ms)Running coprocessor post-close hooks at 1733770445757 (+4 ms)Closed at 1733770445757 2024-12-09T18:54:05,757 DEBUG [RS_CLOSE_META-regionserver/a437f8b9ba7d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T18:54:05,819 INFO [regionserver/a437f8b9ba7d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T18:54:05,821 INFO [regionserver/a437f8b9ba7d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T18:54:05,929 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(976): stopping server a437f8b9ba7d,35449,1733770403924; all regions closed. 2024-12-09T18:54:05,930 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(976): stopping server a437f8b9ba7d,37501,1733770404084; all regions closed. 2024-12-09T18:54:05,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741834_1010 (size=3621) 2024-12-09T18:54:05,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741893_1071 (size=2625) 2024-12-09T18:54:05,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741893_1071 (size=2625) 2024-12-09T18:54:05,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741834_1010 (size=3621) 2024-12-09T18:54:05,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741834_1010 (size=3621) 2024-12-09T18:54:05,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741893_1071 (size=2625) 2024-12-09T18:54:05,938 DEBUG [RS:2;a437f8b9ba7d:37501 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/oldWALs 2024-12-09T18:54:05,938 INFO [RS:2;a437f8b9ba7d:37501 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a437f8b9ba7d%2C37501%2C1733770404084.meta:.meta(num 1733770422890) 2024-12-09T18:54:05,938 DEBUG [RS:0;a437f8b9ba7d:35449 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/oldWALs 2024-12-09T18:54:05,938 INFO [RS:0;a437f8b9ba7d:35449 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a437f8b9ba7d%2C35449%2C1733770403924:(num 1733770405847) 2024-12-09T18:54:05,938 DEBUG [RS:0;a437f8b9ba7d:35449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T18:54:05,938 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T18:54:05,938 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T18:54:05,938 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.ChoreService(370): Chore service for: regionserver/a437f8b9ba7d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T18:54:05,938 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T18:54:05,938 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T18:54:05,938 INFO [regionserver/a437f8b9ba7d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T18:54:05,938 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T18:54:05,938 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T18:54:05,939 INFO [RS:0;a437f8b9ba7d:35449 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35449 2024-12-09T18:54:05,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741833_1009 (size=175) 2024-12-09T18:54:05,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741833_1009 (size=175) 2024-12-09T18:54:05,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741833_1009 (size=175) 2024-12-09T18:54:05,943 DEBUG [RS:2;a437f8b9ba7d:37501 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/oldWALs 2024-12-09T18:54:05,944 INFO [RS:2;a437f8b9ba7d:37501 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a437f8b9ba7d%2C37501%2C1733770404084:(num 1733770405845) 2024-12-09T18:54:05,944 DEBUG [RS:2;a437f8b9ba7d:37501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T18:54:05,944 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T18:54:05,944 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T18:54:05,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T18:54:05,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a437f8b9ba7d,35449,1733770403924 2024-12-09T18:54:05,944 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T18:54:05,944 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.ChoreService(370): Chore service for: regionserver/a437f8b9ba7d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T18:54:05,944 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T18:54:05,944 INFO [RS:2;a437f8b9ba7d:37501 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37501 2024-12-09T18:54:05,945 INFO [regionserver/a437f8b9ba7d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T18:54:05,952 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a437f8b9ba7d,35449,1733770403924] 2024-12-09T18:54:05,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a437f8b9ba7d,37501,1733770404084 2024-12-09T18:54:05,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T18:54:05,961 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T18:54:05,969 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a437f8b9ba7d,35449,1733770403924 already deleted, retry=false 2024-12-09T18:54:05,969 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a437f8b9ba7d,35449,1733770403924 expired; onlineServers=1 2024-12-09T18:54:05,977 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a437f8b9ba7d,37501,1733770404084] 2024-12-09T18:54:05,985 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a437f8b9ba7d,37501,1733770404084 already deleted, retry=false 2024-12-09T18:54:05,986 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a437f8b9ba7d,37501,1733770404084 expired; onlineServers=0 2024-12-09T18:54:05,986 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a437f8b9ba7d,35375,1733770403171' ***** 2024-12-09T18:54:05,986 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T18:54:05,986 INFO [M:0;a437f8b9ba7d:35375 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T18:54:05,986 INFO [M:0;a437f8b9ba7d:35375 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T18:54:05,986 DEBUG [M:0;a437f8b9ba7d:35375 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T18:54:05,986 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T18:54:05,986 DEBUG [M:0;a437f8b9ba7d:35375 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T18:54:05,986 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.small.0-1733770405491 {}] cleaner.HFileCleaner(306): Exit Thread[master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.small.0-1733770405491,5,FailOnTimeoutGroup] 2024-12-09T18:54:05,986 DEBUG [master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.large.0-1733770405489 {}] cleaner.HFileCleaner(306): Exit Thread[master/a437f8b9ba7d:0:becomeActiveMaster-HFileCleaner.large.0-1733770405489,5,FailOnTimeoutGroup] 2024-12-09T18:54:05,987 INFO [M:0;a437f8b9ba7d:35375 {}] hbase.ChoreService(370): Chore service for: master/a437f8b9ba7d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T18:54:05,987 INFO [M:0;a437f8b9ba7d:35375 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T18:54:05,987 DEBUG [M:0;a437f8b9ba7d:35375 {}] master.HMaster(1795): Stopping service threads 2024-12-09T18:54:05,987 INFO [M:0;a437f8b9ba7d:35375 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T18:54:05,987 INFO [M:0;a437f8b9ba7d:35375 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T18:54:05,988 INFO [M:0;a437f8b9ba7d:35375 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T18:54:05,988 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T18:54:05,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T18:54:05,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T18:54:05,994 DEBUG [M:0;a437f8b9ba7d:35375 {}] zookeeper.ZKUtil(347): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T18:54:05,995 WARN [M:0;a437f8b9ba7d:35375 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T18:54:05,996 INFO [M:0;a437f8b9ba7d:35375 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/.lastflushedseqids 2024-12-09T18:54:06,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741944_1124 (size=119) 2024-12-09T18:54:06,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741944_1124 (size=119) 2024-12-09T18:54:06,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741944_1124 (size=119) 2024-12-09T18:54:06,016 INFO [M:0;a437f8b9ba7d:35375 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T18:54:06,016 INFO [M:0;a437f8b9ba7d:35375 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T18:54:06,016 DEBUG [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T18:54:06,016 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T18:54:06,016 DEBUG [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T18:54:06,016 DEBUG [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T18:54:06,016 DEBUG [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T18:54:06,016 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-12-09T18:54:06,029 DEBUG [M:0;a437f8b9ba7d:35375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f256852a7edd418189b6d455a04ab9ad is 82, key is hbase:meta,,1/info:regioninfo/1733770423096/Put/seqid=0 2024-12-09T18:54:06,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741945_1125 (size=6063) 2024-12-09T18:54:06,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741945_1125 (size=6063) 2024-12-09T18:54:06,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741945_1125 (size=6063) 2024-12-09T18:54:06,035 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f256852a7edd418189b6d455a04ab9ad 2024-12-09T18:54:06,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T18:54:06,052 INFO [RS:0;a437f8b9ba7d:35449 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T18:54:06,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35449-0x1000c23f6fa0001, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T18:54:06,052 INFO [RS:0;a437f8b9ba7d:35449 {}] regionserver.HRegionServer(1031): Exiting; stopping=a437f8b9ba7d,35449,1733770403924; zookeeper connection closed. 2024-12-09T18:54:06,053 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@697e84f8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@697e84f8 2024-12-09T18:54:06,054 DEBUG [M:0;a437f8b9ba7d:35375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f5c2b8ba5f745feb1099f9b1fae8c65 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733770420058/Put/seqid=0 2024-12-09T18:54:06,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741946_1126 (size=7906) 2024-12-09T18:54:06,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741946_1126 (size=7906) 2024-12-09T18:54:06,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741946_1126 (size=7906) 2024-12-09T18:54:06,060 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f5c2b8ba5f745feb1099f9b1fae8c65 2024-12-09T18:54:06,065 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f5c2b8ba5f745feb1099f9b1fae8c65 2024-12-09T18:54:06,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T18:54:06,077 INFO [RS:2;a437f8b9ba7d:37501 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T18:54:06,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37501-0x1000c23f6fa0003, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T18:54:06,077 INFO [RS:2;a437f8b9ba7d:37501 {}] regionserver.HRegionServer(1031): Exiting; stopping=a437f8b9ba7d,37501,1733770404084; zookeeper connection closed. 2024-12-09T18:54:06,078 DEBUG [M:0;a437f8b9ba7d:35375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7346959ef13045eeb77d14cae483bd4c is 69, key is a437f8b9ba7d,35449,1733770403924/rs:state/1733770405539/Put/seqid=0 2024-12-09T18:54:06,078 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@c1e1968 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@c1e1968 2024-12-09T18:54:06,078 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T18:54:06,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741947_1127 (size=5440) 2024-12-09T18:54:06,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741947_1127 (size=5440) 2024-12-09T18:54:06,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741947_1127 (size=5440) 2024-12-09T18:54:06,084 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7346959ef13045eeb77d14cae483bd4c 2024-12-09T18:54:06,087 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7346959ef13045eeb77d14cae483bd4c 2024-12-09T18:54:06,088 DEBUG [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f256852a7edd418189b6d455a04ab9ad as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f256852a7edd418189b6d455a04ab9ad 2024-12-09T18:54:06,093 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f256852a7edd418189b6d455a04ab9ad, entries=14, sequenceid=207, filesize=5.9 K 2024-12-09T18:54:06,095 DEBUG [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4f5c2b8ba5f745feb1099f9b1fae8c65 as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f5c2b8ba5f745feb1099f9b1fae8c65 2024-12-09T18:54:06,099 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f5c2b8ba5f745feb1099f9b1fae8c65 2024-12-09T18:54:06,099 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4f5c2b8ba5f745feb1099f9b1fae8c65, entries=21, sequenceid=207, filesize=7.7 K 2024-12-09T18:54:06,100 DEBUG [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7346959ef13045eeb77d14cae483bd4c as hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7346959ef13045eeb77d14cae483bd4c 2024-12-09T18:54:06,105 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7346959ef13045eeb77d14cae483bd4c 2024-12-09T18:54:06,105 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41577/user/jenkins/test-data/fd26c66f-bbda-5ffd-3b53-720784a3b5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7346959ef13045eeb77d14cae483bd4c, entries=3, sequenceid=207, filesize=5.3 K 2024-12-09T18:54:06,106 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=207, compaction requested=false 2024-12-09T18:54:06,107 INFO [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T18:54:06,107 DEBUG [M:0;a437f8b9ba7d:35375 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733770446016Disabling compacts and flushes for region at 1733770446016Disabling writes for close at 1733770446016Obtaining lock to block concurrent updates at 1733770446016Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733770446016Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1733770446016Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733770446017 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733770446017Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733770446029 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733770446029Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733770446039 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733770446054 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733770446054Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733770446065 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733770446077 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733770446077Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f092f68: reopening flushed file at 1733770446088 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47b6f3b1: reopening flushed file at 1733770446094 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@677becd1: reopening flushed file at 1733770446099 (+5 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=207, compaction requested=false at 1733770446106 (+7 ms)Writing region close event to WAL at 1733770446107 (+1 ms)Closed at 1733770446107 2024-12-09T18:54:06,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37407 is added to blk_1073741830_1006 (size=102150) 2024-12-09T18:54:06,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36059 is added to blk_1073741830_1006 (size=102150) 2024-12-09T18:54:06,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38029 is added to blk_1073741830_1006 (size=102150) 2024-12-09T18:54:06,110 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T18:54:06,110 INFO [M:0;a437f8b9ba7d:35375 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T18:54:06,110 INFO [M:0;a437f8b9ba7d:35375 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35375 2024-12-09T18:54:06,111 INFO [M:0;a437f8b9ba7d:35375 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T18:54:06,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T18:54:06,294 INFO [M:0;a437f8b9ba7d:35375 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T18:54:06,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35375-0x1000c23f6fa0000, quorum=127.0.0.1:52975, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T18:54:06,302 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770434239 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770434239 (inode 16688) Holder DFSClient_NONMAPREDUCE_-339328517_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733770424914/wal.1733770434239 (inode 16688) Holder DFSClient_NONMAPREDUCE_-339328517_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T18:54:06,305 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424747 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424747 (inode 16665) Holder DFSClient_NONMAPREDUCE_-339328517_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733770424407/wal.1733770424747 (inode 16665) Holder DFSClient_NONMAPREDUCE_-339328517_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T18:54:06,306 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733770415107/wal.1733770415247 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:54:06,310 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770435219 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770435219 (inode 16714) Holder DFSClient_NONMAPREDUCE_-339328517_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733770434538/wal.1733770435219 (inode 16714) Holder DFSClient_NONMAPREDUCE_-339328517_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T18:54:06,311 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733770435408/wal.1733770435518 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:54:06,311 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733770440441/wal.1733770440701 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:54:06,312 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733770408262/wal.1733770408386 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T18:54:06,314 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal.1733770408074 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal.1733770408074 (inode 16485) Holder DFSClient_NONMAPREDUCE_-339328517_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733770407614/wal.1733770408074 (inode 16485) Holder DFSClient_NONMAPREDUCE_-339328517_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T18:54:06,318 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal.1733770407312 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal.1733770407312 (inode 16462) Holder DFSClient_NONMAPREDUCE_-339328517_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733770406861/wal.1733770407312 (inode 16462) Holder DFSClient_NONMAPREDUCE_-339328517_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T18:54:06,341 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26f8cb79{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T18:54:06,344 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10705e91{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T18:54:06,344 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T18:54:06,345 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16c07ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T18:54:06,345 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4bbcb0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/hadoop.log.dir/,STOPPED} 2024-12-09T18:54:06,349 WARN [BP-1882736132-172.17.0.2-1733770399228 heartbeating to localhost/127.0.0.1:41577 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T18:54:06,349 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T18:54:06,349 WARN [BP-1882736132-172.17.0.2-1733770399228 heartbeating to localhost/127.0.0.1:41577 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1882736132-172.17.0.2-1733770399228 (Datanode Uuid d29cd452-4b12-4833-9e77-4b10cf55769f) service to localhost/127.0.0.1:41577 2024-12-09T18:54:06,349 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T18:54:06,350 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data5/current/BP-1882736132-172.17.0.2-1733770399228 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T18:54:06,350 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data6/current/BP-1882736132-172.17.0.2-1733770399228 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T18:54:06,351 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T18:54:06,367 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4464725f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T18:54:06,367 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11808715{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T18:54:06,367 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T18:54:06,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f9065a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T18:54:06,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@243ebd61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/hadoop.log.dir/,STOPPED} 2024-12-09T18:54:06,372 WARN [BP-1882736132-172.17.0.2-1733770399228 heartbeating to localhost/127.0.0.1:41577 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T18:54:06,372 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T18:54:06,372 WARN [BP-1882736132-172.17.0.2-1733770399228 heartbeating to localhost/127.0.0.1:41577 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1882736132-172.17.0.2-1733770399228 (Datanode Uuid b8ae8be4-88db-477b-9534-d9ab69a4f121) service to localhost/127.0.0.1:41577 2024-12-09T18:54:06,372 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T18:54:06,373 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data3/current/BP-1882736132-172.17.0.2-1733770399228 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T18:54:06,373 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data4/current/BP-1882736132-172.17.0.2-1733770399228 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T18:54:06,374 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T18:54:06,388 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@17d1a8d5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T18:54:06,389 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4afdb36e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T18:54:06,389 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T18:54:06,389 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fcc4f36{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T18:54:06,389 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@126e7def{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/hadoop.log.dir/,STOPPED} 2024-12-09T18:54:06,391 WARN [BP-1882736132-172.17.0.2-1733770399228 heartbeating to localhost/127.0.0.1:41577 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T18:54:06,391 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T18:54:06,391 WARN [BP-1882736132-172.17.0.2-1733770399228 heartbeating to localhost/127.0.0.1:41577 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1882736132-172.17.0.2-1733770399228 (Datanode Uuid 379ba87a-21a3-409d-8cc1-0961b6a04a16) service to localhost/127.0.0.1:41577 2024-12-09T18:54:06,391 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T18:54:06,392 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data1/current/BP-1882736132-172.17.0.2-1733770399228 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T18:54:06,392 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/cluster_a04cc761-37df-b2b7-5ebd-a43610944bdc/data/data2/current/BP-1882736132-172.17.0.2-1733770399228 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T18:54:06,392 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T18:54:06,404 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5608f426{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T18:54:06,405 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@63efd3a6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T18:54:06,405 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T18:54:06,405 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3192a19b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T18:54:06,405 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e7ae814{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f2034b00-9cda-dfc6-46cd-9eb1e8baff42/hadoop.log.dir/,STOPPED} 2024-12-09T18:54:06,418 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T18:54:06,490 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down