2024-12-09 14:46:56,222 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 14:46:56,236 main DEBUG Took 0.011184 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 14:46:56,236 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 14:46:56,237 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 14:46:56,238 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 14:46:56,239 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,244 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 14:46:56,255 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,257 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,257 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,257 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,258 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,259 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,259 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,260 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,260 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,260 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,260 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,261 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,261 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,262 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,262 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,263 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,263 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,263 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,264 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 14:46:56,264 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,264 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 14:46:56,265 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 14:46:56,267 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 14:46:56,268 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 14:46:56,269 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 14:46:56,270 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 14:46:56,270 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 14:46:56,277 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 14:46:56,279 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 14:46:56,281 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 14:46:56,281 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 14:46:56,282 main DEBUG createAppenders(={Console}) 2024-12-09 14:46:56,282 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-09 14:46:56,283 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-09 14:46:56,283 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-09 14:46:56,283 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 14:46:56,284 main DEBUG OutputStream closed 2024-12-09 14:46:56,284 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 14:46:56,284 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 14:46:56,284 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-09 14:46:56,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 14:46:56,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 14:46:56,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 14:46:56,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 14:46:56,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 14:46:56,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 14:46:56,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 14:46:56,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 14:46:56,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 14:46:56,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 14:46:56,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 14:46:56,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 14:46:56,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 14:46:56,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 14:46:56,351 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 14:46:56,351 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 14:46:56,351 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 14:46:56,352 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 14:46:56,354 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 14:46:56,354 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-09 14:46:56,354 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 14:46:56,355 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-09T14:46:56,548 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829 2024-12-09 14:46:56,551 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 14:46:56,551 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T14:46:56,559 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-09T14:46:56,564 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-09T14:46:56,581 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T14:46:56,619 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T14:46:56,619 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T14:46:56,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T14:46:56,643 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd, deleteOnExit=true 2024-12-09T14:46:56,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T14:46:56,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/test.cache.data in system properties and HBase conf 2024-12-09T14:46:56,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T14:46:56,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/hadoop.log.dir in system properties and HBase conf 2024-12-09T14:46:56,645 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T14:46:56,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T14:46:56,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T14:46:56,716 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T14:46:56,794 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T14:46:56,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T14:46:56,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T14:46:56,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T14:46:56,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T14:46:56,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T14:46:56,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T14:46:56,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T14:46:56,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T14:46:56,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T14:46:56,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/nfs.dump.dir in system properties and HBase conf 2024-12-09T14:46:56,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/java.io.tmpdir in system properties and HBase conf 2024-12-09T14:46:56,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T14:46:56,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T14:46:56,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T14:46:57,719 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T14:46:57,787 INFO [Time-limited test {}] log.Log(170): Logging initialized @2171ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T14:46:57,853 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T14:46:57,907 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T14:46:57,928 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T14:46:57,928 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T14:46:57,929 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T14:46:57,946 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T14:46:57,948 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/hadoop.log.dir/,AVAILABLE} 2024-12-09T14:46:57,949 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T14:46:58,107 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/java.io.tmpdir/jetty-localhost-45727-hadoop-hdfs-3_4_1-tests_jar-_-any-9119186482021623646/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T14:46:58,112 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:45727} 2024-12-09T14:46:58,112 INFO [Time-limited test {}] server.Server(415): Started @2496ms 2024-12-09T14:46:58,594 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T14:46:58,600 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T14:46:58,601 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T14:46:58,601 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T14:46:58,602 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T14:46:58,602 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/hadoop.log.dir/,AVAILABLE} 2024-12-09T14:46:58,603 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T14:46:58,696 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/java.io.tmpdir/jetty-localhost-44389-hadoop-hdfs-3_4_1-tests_jar-_-any-12796207921212413032/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:46:58,696 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:44389} 2024-12-09T14:46:58,696 INFO [Time-limited test {}] server.Server(415): Started @3080ms 2024-12-09T14:46:58,739 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T14:46:58,837 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T14:46:58,842 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T14:46:58,843 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T14:46:58,843 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T14:46:58,844 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T14:46:58,845 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/hadoop.log.dir/,AVAILABLE} 2024-12-09T14:46:58,846 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T14:46:58,940 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/java.io.tmpdir/jetty-localhost-33689-hadoop-hdfs-3_4_1-tests_jar-_-any-6242173474284261007/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:46:58,941 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:33689} 2024-12-09T14:46:58,941 INFO [Time-limited test {}] server.Server(415): Started @3325ms 2024-12-09T14:46:58,943 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T14:46:58,977 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T14:46:58,981 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T14:46:58,983 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T14:46:58,983 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T14:46:58,983 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T14:46:58,984 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/hadoop.log.dir/,AVAILABLE} 2024-12-09T14:46:58,984 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T14:46:59,079 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/java.io.tmpdir/jetty-localhost-45609-hadoop-hdfs-3_4_1-tests_jar-_-any-497453185739414836/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:46:59,080 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:45609} 2024-12-09T14:46:59,080 INFO [Time-limited test {}] server.Server(415): Started @3464ms 2024-12-09T14:46:59,082 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T14:47:00,087 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data3/current/BP-2135414708-172.17.0.2-1733755617276/current, will proceed with Du for space computation calculation, 2024-12-09T14:47:00,087 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data4/current/BP-2135414708-172.17.0.2-1733755617276/current, will proceed with Du for space computation calculation, 2024-12-09T14:47:00,087 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data1/current/BP-2135414708-172.17.0.2-1733755617276/current, will proceed with Du for space computation calculation, 2024-12-09T14:47:00,087 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data2/current/BP-2135414708-172.17.0.2-1733755617276/current, will proceed with Du for space computation calculation, 2024-12-09T14:47:00,120 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T14:47:00,120 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T14:47:00,160 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data5/current/BP-2135414708-172.17.0.2-1733755617276/current, will proceed with Du for space computation calculation, 2024-12-09T14:47:00,161 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data6/current/BP-2135414708-172.17.0.2-1733755617276/current, will proceed with Du for space computation calculation, 2024-12-09T14:47:00,166 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf954553bcf218d97 with lease ID 0xe41f4de837572586: Processing first storage report for DS-c0e9d82e-4339-49b1-9400-e7877e2718e9 from datanode DatanodeRegistration(127.0.0.1:46775, datanodeUuid=8b469460-f07d-4e65-a333-3cf61e240627, infoPort=34373, infoSecurePort=0, ipcPort=42081, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276) 2024-12-09T14:47:00,168 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf954553bcf218d97 with lease ID 0xe41f4de837572586: from storage DS-c0e9d82e-4339-49b1-9400-e7877e2718e9 node DatanodeRegistration(127.0.0.1:46775, datanodeUuid=8b469460-f07d-4e65-a333-3cf61e240627, infoPort=34373, infoSecurePort=0, ipcPort=42081, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-09T14:47:00,168 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d7219c5e9332cd4 with lease ID 0xe41f4de837572587: Processing first storage report for DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc from datanode DatanodeRegistration(127.0.0.1:33317, datanodeUuid=a8fc4096-befb-456e-84a7-962edeb9a8d6, infoPort=35879, infoSecurePort=0, ipcPort=46141, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276) 2024-12-09T14:47:00,169 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d7219c5e9332cd4 with lease ID 0xe41f4de837572587: from storage DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc node DatanodeRegistration(127.0.0.1:33317, datanodeUuid=a8fc4096-befb-456e-84a7-962edeb9a8d6, infoPort=35879, infoSecurePort=0, ipcPort=46141, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T14:47:00,169 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf954553bcf218d97 with lease ID 0xe41f4de837572586: Processing first storage report for DS-ecff3e80-c1c0-419e-bde3-a823780d005c from datanode DatanodeRegistration(127.0.0.1:46775, datanodeUuid=8b469460-f07d-4e65-a333-3cf61e240627, infoPort=34373, infoSecurePort=0, ipcPort=42081, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276) 2024-12-09T14:47:00,169 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf954553bcf218d97 with lease ID 0xe41f4de837572586: from storage DS-ecff3e80-c1c0-419e-bde3-a823780d005c node DatanodeRegistration(127.0.0.1:46775, datanodeUuid=8b469460-f07d-4e65-a333-3cf61e240627, infoPort=34373, infoSecurePort=0, ipcPort=42081, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T14:47:00,169 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d7219c5e9332cd4 with lease ID 0xe41f4de837572587: Processing first storage report for DS-2a0ef1ba-b66a-4c5c-880e-70bf7800e9c0 from datanode DatanodeRegistration(127.0.0.1:33317, datanodeUuid=a8fc4096-befb-456e-84a7-962edeb9a8d6, infoPort=35879, infoSecurePort=0, ipcPort=46141, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276) 2024-12-09T14:47:00,170 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d7219c5e9332cd4 with lease ID 0xe41f4de837572587: from storage DS-2a0ef1ba-b66a-4c5c-880e-70bf7800e9c0 node DatanodeRegistration(127.0.0.1:33317, datanodeUuid=a8fc4096-befb-456e-84a7-962edeb9a8d6, infoPort=35879, infoSecurePort=0, ipcPort=46141, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T14:47:00,178 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T14:47:00,183 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc315f5859da05ac8 with lease ID 0xe41f4de837572588: Processing first storage report for DS-881664bb-1345-45a8-9a41-c28a0e496621 from datanode DatanodeRegistration(127.0.0.1:37163, datanodeUuid=c418ed6f-3494-4a7f-b949-673db0ef1ea1, infoPort=46429, infoSecurePort=0, ipcPort=37069, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276) 2024-12-09T14:47:00,184 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc315f5859da05ac8 with lease ID 0xe41f4de837572588: from storage DS-881664bb-1345-45a8-9a41-c28a0e496621 node DatanodeRegistration(127.0.0.1:37163, datanodeUuid=c418ed6f-3494-4a7f-b949-673db0ef1ea1, infoPort=46429, infoSecurePort=0, ipcPort=37069, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T14:47:00,184 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc315f5859da05ac8 with lease ID 0xe41f4de837572588: Processing first storage report for DS-3764460a-03ab-476e-8fc8-5439a143bef6 from datanode DatanodeRegistration(127.0.0.1:37163, datanodeUuid=c418ed6f-3494-4a7f-b949-673db0ef1ea1, infoPort=46429, infoSecurePort=0, ipcPort=37069, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276) 2024-12-09T14:47:00,184 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc315f5859da05ac8 with lease ID 0xe41f4de837572588: from storage DS-3764460a-03ab-476e-8fc8-5439a143bef6 node DatanodeRegistration(127.0.0.1:37163, datanodeUuid=c418ed6f-3494-4a7f-b949-673db0ef1ea1, infoPort=46429, infoSecurePort=0, ipcPort=37069, storageInfo=lv=-57;cid=testClusterID;nsid=2022222826;c=1733755617276), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T14:47:00,202 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829 2024-12-09T14:47:00,262 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/zookeeper_0, clientPort=58381, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T14:47:00,270 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58381 2024-12-09T14:47:00,280 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:00,284 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:00,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741825_1001 (size=7) 2024-12-09T14:47:00,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741825_1001 (size=7) 2024-12-09T14:47:00,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741825_1001 (size=7) 2024-12-09T14:47:00,893 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f with version=8 2024-12-09T14:47:00,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/hbase-staging 2024-12-09T14:47:01,165 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:47:01,173 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,174 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,178 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:47:01,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:47:01,290 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T14:47:01,338 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T14:47:01,346 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T14:47:01,349 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:47:01,370 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 23483 (auto-detected) 2024-12-09T14:47:01,371 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T14:47:01,387 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40599 2024-12-09T14:47:01,405 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40599 connecting to ZooKeeper ensemble=127.0.0.1:58381 2024-12-09T14:47:01,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405990x0, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:47:01,525 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40599-0x1000b190a330000 connected 2024-12-09T14:47:01,625 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:01,632 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:01,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:47:01,645 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f, hbase.cluster.distributed=false 2024-12-09T14:47:01,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:47:01,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40599 2024-12-09T14:47:01,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40599 2024-12-09T14:47:01,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40599 2024-12-09T14:47:01,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40599 2024-12-09T14:47:01,672 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40599 2024-12-09T14:47:01,761 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:47:01,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,763 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,763 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:47:01,763 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,763 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:47:01,766 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:47:01,768 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:47:01,769 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37129 2024-12-09T14:47:01,771 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37129 connecting to ZooKeeper ensemble=127.0.0.1:58381 2024-12-09T14:47:01,772 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:01,774 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:01,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:371290x0, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:47:01,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:371290x0, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:47:01,791 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37129-0x1000b190a330001 connected 2024-12-09T14:47:01,795 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:47:01,803 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:47:01,805 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T14:47:01,811 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:47:01,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37129 2024-12-09T14:47:01,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37129 2024-12-09T14:47:01,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37129 2024-12-09T14:47:01,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37129 2024-12-09T14:47:01,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37129 2024-12-09T14:47:01,827 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:47:01,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,827 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,828 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:47:01,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:47:01,828 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:47:01,828 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:47:01,829 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43689 2024-12-09T14:47:01,830 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43689 connecting to ZooKeeper ensemble=127.0.0.1:58381 2024-12-09T14:47:01,831 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:01,833 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:01,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436890x0, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:47:01,848 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:436890x0, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:47:01,848 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43689-0x1000b190a330002 connected 2024-12-09T14:47:01,848 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:47:01,849 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:47:01,850 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T14:47:01,852 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:47:01,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43689 2024-12-09T14:47:01,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43689 2024-12-09T14:47:01,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43689 2024-12-09T14:47:01,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43689 2024-12-09T14:47:01,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43689 2024-12-09T14:47:01,871 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2491923345a5:0 server-side Connection retries=45 2024-12-09T14:47:01,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,871 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T14:47:01,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T14:47:01,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T14:47:01,872 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T14:47:01,872 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T14:47:01,873 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44703 2024-12-09T14:47:01,874 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44703 connecting to ZooKeeper ensemble=127.0.0.1:58381 2024-12-09T14:47:01,875 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:01,877 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:01,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447030x0, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T14:47:01,891 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44703-0x1000b190a330003 connected 2024-12-09T14:47:01,891 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:47:01,891 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T14:47:01,892 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T14:47:01,893 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T14:47:01,894 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T14:47:01,895 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44703 2024-12-09T14:47:01,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44703 2024-12-09T14:47:01,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44703 2024-12-09T14:47:01,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44703 2024-12-09T14:47:01,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44703 2024-12-09T14:47:01,916 DEBUG [M:0;2491923345a5:40599 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2491923345a5:40599 2024-12-09T14:47:01,917 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2491923345a5,40599,1733755621019 2024-12-09T14:47:01,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:47:01,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:47:01,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:47:01,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:47:01,935 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2491923345a5,40599,1733755621019 2024-12-09T14:47:01,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T14:47:01,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T14:47:01,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:01,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T14:47:01,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:01,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:01,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:01,965 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T14:47:01,966 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2491923345a5,40599,1733755621019 from backup master directory 2024-12-09T14:47:01,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:47:01,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:47:01,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:47:01,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2491923345a5,40599,1733755621019 2024-12-09T14:47:01,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T14:47:01,975 WARN [master/2491923345a5:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:47:01,975 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2491923345a5,40599,1733755621019 2024-12-09T14:47:01,978 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T14:47:01,980 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T14:47:02,031 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/hbase.id] with ID: 20c7aab3-df44-4d3a-95f0-b2f8c0a2e453 2024-12-09T14:47:02,031 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/.tmp/hbase.id 2024-12-09T14:47:02,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741826_1002 (size=42) 2024-12-09T14:47:02,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741826_1002 (size=42) 2024-12-09T14:47:02,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741826_1002 (size=42) 2024-12-09T14:47:02,045 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/.tmp/hbase.id]:[hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/hbase.id] 2024-12-09T14:47:02,094 INFO [master/2491923345a5:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T14:47:02,100 INFO [master/2491923345a5:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T14:47:02,115 INFO [master/2491923345a5:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-09T14:47:02,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,147 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741827_1003 (size=196) 2024-12-09T14:47:02,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741827_1003 (size=196) 2024-12-09T14:47:02,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741827_1003 (size=196) 2024-12-09T14:47:02,185 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T14:47:02,187 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T14:47:02,192 INFO [master/2491923345a5:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:02,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741828_1004 (size=1189) 2024-12-09T14:47:02,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741828_1004 (size=1189) 2024-12-09T14:47:02,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741828_1004 (size=1189) 2024-12-09T14:47:02,236 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store 2024-12-09T14:47:02,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741829_1005 (size=34) 2024-12-09T14:47:02,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741829_1005 (size=34) 2024-12-09T14:47:02,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741829_1005 (size=34) 2024-12-09T14:47:02,263 INFO [master/2491923345a5:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T14:47:02,266 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:02,267 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T14:47:02,267 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:47:02,268 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:47:02,269 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T14:47:02,269 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:47:02,269 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:47:02,270 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733755622267Disabling compacts and flushes for region at 1733755622267Disabling writes for close at 1733755622269 (+2 ms)Writing region close event to WAL at 1733755622269Closed at 1733755622269 2024-12-09T14:47:02,272 WARN [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/.initializing 2024-12-09T14:47:02,272 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/WALs/2491923345a5,40599,1733755621019 2024-12-09T14:47:02,279 INFO [master/2491923345a5:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:02,292 INFO [master/2491923345a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2491923345a5%2C40599%2C1733755621019, suffix=, logDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/WALs/2491923345a5,40599,1733755621019, archiveDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/oldWALs, maxLogs=10 2024-12-09T14:47:02,317 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/WALs/2491923345a5,40599,1733755621019/2491923345a5%2C40599%2C1733755621019.1733755622296, exclude list is [], retry=0 2024-12-09T14:47:02,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:02,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:02,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:02,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:02,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T14:47:02,370 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-09T14:47:02,423 INFO [master/2491923345a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/WALs/2491923345a5,40599,1733755621019/2491923345a5%2C40599%2C1733755621019.1733755622296 2024-12-09T14:47:02,424 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:02,424 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:02,425 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:02,427 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,428 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T14:47:02,483 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:02,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:47:02,486 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,489 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T14:47:02,489 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:02,490 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:02,490 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,493 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T14:47:02,493 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:02,494 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:02,494 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T14:47:02,497 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:02,498 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:02,498 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,502 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,503 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,507 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,508 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,511 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:47:02,515 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T14:47:02,520 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:02,521 INFO [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70454829, jitterRate=0.04985876381397247}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:47:02,527 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733755622439Initializing all the Stores at 1733755622441 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733755622442 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755622442Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755622442Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755622442Cleaning up temporary data from old regions at 1733755622508 (+66 ms)Region opened successfully at 1733755622527 (+19 ms) 2024-12-09T14:47:02,528 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T14:47:02,561 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754c3069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:47:02,590 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T14:47:02,600 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T14:47:02,601 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T14:47:02,603 INFO [master/2491923345a5:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T14:47:02,604 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T14:47:02,609 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T14:47:02,609 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T14:47:02,634 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T14:47:02,641 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T14:47:02,690 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T14:47:02,695 INFO [master/2491923345a5:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T14:47:02,697 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T14:47:02,706 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T14:47:02,709 INFO [master/2491923345a5:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T14:47:02,713 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T14:47:02,723 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T14:47:02,725 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T14:47:02,731 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T14:47:02,752 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T14:47:02,763 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T14:47:02,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:47:02,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:47:02,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:47:02,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T14:47:02,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,778 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2491923345a5,40599,1733755621019, sessionid=0x1000b190a330000, setting cluster-up flag (Was=false) 2024-12-09T14:47:02,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,831 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T14:47:02,833 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2491923345a5,40599,1733755621019 2024-12-09T14:47:02,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:02,881 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T14:47:02,883 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2491923345a5,40599,1733755621019 2024-12-09T14:47:02,890 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T14:47:02,901 INFO [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(746): ClusterId : 20c7aab3-df44-4d3a-95f0-b2f8c0a2e453 2024-12-09T14:47:02,901 INFO [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(746): ClusterId : 20c7aab3-df44-4d3a-95f0-b2f8c0a2e453 2024-12-09T14:47:02,901 INFO [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(746): ClusterId : 20c7aab3-df44-4d3a-95f0-b2f8c0a2e453 2024-12-09T14:47:02,903 DEBUG [RS:1;2491923345a5:43689 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:47:02,903 DEBUG [RS:2;2491923345a5:44703 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:47:02,903 DEBUG [RS:0;2491923345a5:37129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T14:47:02,925 DEBUG [RS:2;2491923345a5:44703 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:47:02,925 DEBUG [RS:0;2491923345a5:37129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:47:02,925 DEBUG [RS:1;2491923345a5:43689 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T14:47:02,926 DEBUG [RS:2;2491923345a5:44703 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:47:02,926 DEBUG [RS:1;2491923345a5:43689 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:47:02,926 DEBUG [RS:0;2491923345a5:37129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T14:47:02,933 DEBUG [RS:1;2491923345a5:43689 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:47:02,933 DEBUG [RS:2;2491923345a5:44703 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:47:02,933 DEBUG [RS:0;2491923345a5:37129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T14:47:02,933 DEBUG [RS:1;2491923345a5:43689 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b38065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:47:02,933 DEBUG [RS:2;2491923345a5:44703 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c632c1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:47:02,933 DEBUG [RS:0;2491923345a5:37129 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c25ddd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2491923345a5/172.17.0.2:0 2024-12-09T14:47:02,947 DEBUG [RS:2;2491923345a5:44703 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2491923345a5:44703 2024-12-09T14:47:02,950 DEBUG [RS:0;2491923345a5:37129 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2491923345a5:37129 2024-12-09T14:47:02,950 DEBUG [RS:1;2491923345a5:43689 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2491923345a5:43689 2024-12-09T14:47:02,950 INFO [RS:2;2491923345a5:44703 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:47:02,950 INFO [RS:1;2491923345a5:43689 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:47:02,950 INFO [RS:0;2491923345a5:37129 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T14:47:02,950 INFO [RS:2;2491923345a5:44703 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:47:02,951 INFO [RS:0;2491923345a5:37129 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:47:02,951 INFO [RS:1;2491923345a5:43689 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T14:47:02,951 DEBUG [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:47:02,951 DEBUG [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:47:02,951 DEBUG [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T14:47:02,953 INFO [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,40599,1733755621019 with port=44703, startcode=1733755621870 2024-12-09T14:47:02,953 INFO [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,40599,1733755621019 with port=43689, startcode=1733755621826 2024-12-09T14:47:02,953 INFO [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,40599,1733755621019 with port=37129, startcode=1733755621729 2024-12-09T14:47:02,958 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T14:47:02,963 DEBUG [RS:0;2491923345a5:37129 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:47:02,963 DEBUG [RS:2;2491923345a5:44703 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:47:02,963 DEBUG [RS:1;2491923345a5:43689 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T14:47:02,970 INFO [master/2491923345a5:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T14:47:02,979 INFO [master/2491923345a5:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T14:47:02,985 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2491923345a5,40599,1733755621019 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T14:47:02,993 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43975, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:47:02,993 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51141, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:47:02,993 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40583, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T14:47:02,994 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:47:02,994 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:47:02,994 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:47:02,994 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2491923345a5:0, corePoolSize=5, maxPoolSize=5 2024-12-09T14:47:02,995 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2491923345a5:0, corePoolSize=10, maxPoolSize=10 2024-12-09T14:47:02,995 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:02,995 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:47:02,995 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,000 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T14:47:03,001 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:47:03,001 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T14:47:03,002 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733755653002 2024-12-09T14:47:03,004 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T14:47:03,005 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T14:47:03,006 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T14:47:03,007 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:03,007 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T14:47:03,007 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T14:47:03,009 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T14:47:03,010 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T14:47:03,010 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T14:47:03,010 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T14:47:03,011 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,015 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T14:47:03,016 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T14:47:03,017 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T14:47:03,019 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T14:47:03,019 INFO [master/2491923345a5:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T14:47:03,021 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.large.0-1733755623020,5,FailOnTimeoutGroup] 2024-12-09T14:47:03,023 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.small.0-1733755623021,5,FailOnTimeoutGroup] 2024-12-09T14:47:03,023 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,023 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T14:47:03,056 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,056 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741831_1007 (size=1321) 2024-12-09T14:47:03,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741831_1007 (size=1321) 2024-12-09T14:47:03,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741831_1007 (size=1321) 2024-12-09T14:47:03,060 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T14:47:03,060 DEBUG [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T14:47:03,060 DEBUG [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T14:47:03,060 DEBUG [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T14:47:03,060 WARN [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T14:47:03,060 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f 2024-12-09T14:47:03,060 WARN [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T14:47:03,060 WARN [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T14:47:03,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741832_1008 (size=32) 2024-12-09T14:47:03,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741832_1008 (size=32) 2024-12-09T14:47:03,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741832_1008 (size=32) 2024-12-09T14:47:03,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:03,075 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T14:47:03,078 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T14:47:03,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:03,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:47:03,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T14:47:03,081 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T14:47:03,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:03,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:47:03,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T14:47:03,086 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T14:47:03,086 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:03,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:47:03,088 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T14:47:03,091 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T14:47:03,091 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:03,092 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:47:03,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T14:47:03,094 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740 2024-12-09T14:47:03,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740 2024-12-09T14:47:03,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T14:47:03,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T14:47:03,098 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:47:03,101 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T14:47:03,105 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:03,106 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71742298, jitterRate=0.06904354691505432}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:47:03,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733755623073Initializing all the Stores at 1733755623075 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733755623075Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733755623075Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755623075Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733755623075Cleaning up temporary data from old regions at 1733755623097 (+22 ms)Region opened successfully at 1733755623108 (+11 ms) 2024-12-09T14:47:03,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T14:47:03,108 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T14:47:03,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T14:47:03,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T14:47:03,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T14:47:03,110 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T14:47:03,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733755623108Disabling compacts and flushes for region at 1733755623108Disabling writes for close at 1733755623108Writing region close event to WAL at 1733755623109 (+1 ms)Closed at 1733755623110 (+1 ms) 2024-12-09T14:47:03,113 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:47:03,113 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T14:47:03,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T14:47:03,126 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T14:47:03,130 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T14:47:03,162 INFO [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,40599,1733755621019 with port=44703, startcode=1733755621870 2024-12-09T14:47:03,162 INFO [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,40599,1733755621019 with port=37129, startcode=1733755621729 2024-12-09T14:47:03,162 INFO [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(2659): reportForDuty to master=2491923345a5,40599,1733755621019 with port=43689, startcode=1733755621826 2024-12-09T14:47:03,164 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2491923345a5,37129,1733755621729 2024-12-09T14:47:03,166 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(517): Registering regionserver=2491923345a5,37129,1733755621729 2024-12-09T14:47:03,172 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2491923345a5,44703,1733755621870 2024-12-09T14:47:03,172 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(517): Registering regionserver=2491923345a5,44703,1733755621870 2024-12-09T14:47:03,173 DEBUG [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f 2024-12-09T14:47:03,173 DEBUG [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35639 2024-12-09T14:47:03,173 DEBUG [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:47:03,176 DEBUG [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f 2024-12-09T14:47:03,176 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2491923345a5,43689,1733755621826 2024-12-09T14:47:03,176 DEBUG [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35639 2024-12-09T14:47:03,176 DEBUG [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:47:03,176 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40599 {}] master.ServerManager(517): Registering regionserver=2491923345a5,43689,1733755621826 2024-12-09T14:47:03,179 DEBUG [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f 2024-12-09T14:47:03,179 DEBUG [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35639 2024-12-09T14:47:03,179 DEBUG [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T14:47:03,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:47:03,229 DEBUG [RS:0;2491923345a5:37129 {}] zookeeper.ZKUtil(111): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2491923345a5,37129,1733755621729 2024-12-09T14:47:03,229 WARN [RS:0;2491923345a5:37129 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:47:03,230 INFO [RS:0;2491923345a5:37129 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:03,230 DEBUG [RS:1;2491923345a5:43689 {}] zookeeper.ZKUtil(111): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2491923345a5,43689,1733755621826 2024-12-09T14:47:03,230 DEBUG [RS:2;2491923345a5:44703 {}] zookeeper.ZKUtil(111): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2491923345a5,44703,1733755621870 2024-12-09T14:47:03,230 WARN [RS:1;2491923345a5:43689 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:47:03,230 WARN [RS:2;2491923345a5:44703 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T14:47:03,230 DEBUG [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729 2024-12-09T14:47:03,230 INFO [RS:1;2491923345a5:43689 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:03,230 INFO [RS:2;2491923345a5:44703 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:03,230 DEBUG [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,44703,1733755621870 2024-12-09T14:47:03,230 DEBUG [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,43689,1733755621826 2024-12-09T14:47:03,231 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2491923345a5,43689,1733755621826] 2024-12-09T14:47:03,231 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2491923345a5,37129,1733755621729] 2024-12-09T14:47:03,231 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2491923345a5,44703,1733755621870] 2024-12-09T14:47:03,252 INFO [RS:1;2491923345a5:43689 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:47:03,252 INFO [RS:0;2491923345a5:37129 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:47:03,252 INFO [RS:2;2491923345a5:44703 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T14:47:03,266 INFO [RS:1;2491923345a5:43689 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:47:03,266 INFO [RS:2;2491923345a5:44703 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:47:03,267 INFO [RS:0;2491923345a5:37129 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T14:47:03,273 INFO [RS:1;2491923345a5:43689 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:47:03,273 INFO [RS:0;2491923345a5:37129 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:47:03,273 INFO [RS:2;2491923345a5:44703 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:47:03,273 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,273 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,273 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,274 INFO [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:47:03,275 INFO [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:47:03,275 INFO [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T14:47:03,280 INFO [RS:1;2491923345a5:43689 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:47:03,280 INFO [RS:2;2491923345a5:44703 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:47:03,280 INFO [RS:0;2491923345a5:37129 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T14:47:03,280 WARN [2491923345a5:40599 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T14:47:03,281 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,281 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,281 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,282 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,282 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:47:03,283 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:47:03,283 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0, corePoolSize=2, maxPoolSize=2 2024-12-09T14:47:03,283 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,283 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,284 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,284 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,284 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,284 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:47:03,284 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2491923345a5:0, corePoolSize=1, maxPoolSize=1 2024-12-09T14:47:03,284 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:47:03,284 DEBUG [RS:1;2491923345a5:43689 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:47:03,284 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:47:03,284 DEBUG [RS:0;2491923345a5:37129 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:47:03,284 DEBUG [RS:2;2491923345a5:44703 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2491923345a5:0, corePoolSize=3, maxPoolSize=3 2024-12-09T14:47:03,292 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,37129,1733755621729-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,43689,1733755621826-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:47:03,292 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,293 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,293 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,44703,1733755621870-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:47:03,314 INFO [RS:1;2491923345a5:43689 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:47:03,314 INFO [RS:0;2491923345a5:37129 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:47:03,315 INFO [RS:2;2491923345a5:44703 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T14:47:03,316 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,37129,1733755621729-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,316 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,44703,1733755621870-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,316 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,43689,1733755621826-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,316 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,316 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,316 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,316 INFO [RS:0;2491923345a5:37129 {}] regionserver.Replication(171): 2491923345a5,37129,1733755621729 started 2024-12-09T14:47:03,316 INFO [RS:1;2491923345a5:43689 {}] regionserver.Replication(171): 2491923345a5,43689,1733755621826 started 2024-12-09T14:47:03,316 INFO [RS:2;2491923345a5:44703 {}] regionserver.Replication(171): 2491923345a5,44703,1733755621870 started 2024-12-09T14:47:03,337 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,338 INFO [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(1482): Serving as 2491923345a5,37129,1733755621729, RpcServer on 2491923345a5/172.17.0.2:37129, sessionid=0x1000b190a330001 2024-12-09T14:47:03,338 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,338 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:03,339 DEBUG [RS:0;2491923345a5:37129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:47:03,339 INFO [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(1482): Serving as 2491923345a5,44703,1733755621870, RpcServer on 2491923345a5/172.17.0.2:44703, sessionid=0x1000b190a330003 2024-12-09T14:47:03,339 INFO [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(1482): Serving as 2491923345a5,43689,1733755621826, RpcServer on 2491923345a5/172.17.0.2:43689, sessionid=0x1000b190a330002 2024-12-09T14:47:03,339 DEBUG [RS:0;2491923345a5:37129 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2491923345a5,37129,1733755621729 2024-12-09T14:47:03,339 DEBUG [RS:1;2491923345a5:43689 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:47:03,339 DEBUG [RS:2;2491923345a5:44703 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T14:47:03,339 DEBUG [RS:0;2491923345a5:37129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,37129,1733755621729' 2024-12-09T14:47:03,339 DEBUG [RS:1;2491923345a5:43689 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2491923345a5,43689,1733755621826 2024-12-09T14:47:03,339 DEBUG [RS:2;2491923345a5:44703 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2491923345a5,44703,1733755621870 2024-12-09T14:47:03,339 DEBUG [RS:1;2491923345a5:43689 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,43689,1733755621826' 2024-12-09T14:47:03,339 DEBUG [RS:2;2491923345a5:44703 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,44703,1733755621870' 2024-12-09T14:47:03,339 DEBUG [RS:1;2491923345a5:43689 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:47:03,339 DEBUG [RS:0;2491923345a5:37129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:47:03,339 DEBUG [RS:2;2491923345a5:44703 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T14:47:03,340 DEBUG [RS:0;2491923345a5:37129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:47:03,340 DEBUG [RS:2;2491923345a5:44703 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:47:03,340 DEBUG [RS:1;2491923345a5:43689 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T14:47:03,341 DEBUG [RS:0;2491923345a5:37129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:47:03,341 DEBUG [RS:1;2491923345a5:43689 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:47:03,341 DEBUG [RS:2;2491923345a5:44703 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T14:47:03,341 DEBUG [RS:0;2491923345a5:37129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:47:03,341 DEBUG [RS:2;2491923345a5:44703 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:47:03,341 DEBUG [RS:1;2491923345a5:43689 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T14:47:03,341 DEBUG [RS:1;2491923345a5:43689 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2491923345a5,43689,1733755621826 2024-12-09T14:47:03,342 DEBUG [RS:2;2491923345a5:44703 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2491923345a5,44703,1733755621870 2024-12-09T14:47:03,342 DEBUG [RS:0;2491923345a5:37129 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2491923345a5,37129,1733755621729 2024-12-09T14:47:03,342 DEBUG [RS:1;2491923345a5:43689 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,43689,1733755621826' 2024-12-09T14:47:03,342 DEBUG [RS:2;2491923345a5:44703 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,44703,1733755621870' 2024-12-09T14:47:03,342 DEBUG [RS:1;2491923345a5:43689 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:47:03,342 DEBUG [RS:0;2491923345a5:37129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2491923345a5,37129,1733755621729' 2024-12-09T14:47:03,342 DEBUG [RS:2;2491923345a5:44703 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:47:03,342 DEBUG [RS:0;2491923345a5:37129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T14:47:03,342 DEBUG [RS:1;2491923345a5:43689 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:47:03,342 DEBUG [RS:0;2491923345a5:37129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:47:03,342 DEBUG [RS:2;2491923345a5:44703 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T14:47:03,343 DEBUG [RS:1;2491923345a5:43689 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:47:03,343 INFO [RS:1;2491923345a5:43689 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:47:03,343 INFO [RS:1;2491923345a5:43689 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:47:03,343 DEBUG [RS:2;2491923345a5:44703 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:47:03,343 INFO [RS:2;2491923345a5:44703 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:47:03,343 INFO [RS:2;2491923345a5:44703 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:47:03,346 DEBUG [RS:0;2491923345a5:37129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T14:47:03,347 INFO [RS:0;2491923345a5:37129 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T14:47:03,347 INFO [RS:0;2491923345a5:37129 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T14:47:03,449 INFO [RS:1;2491923345a5:43689 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:03,449 INFO [RS:0;2491923345a5:37129 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:03,449 INFO [RS:2;2491923345a5:44703 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:03,452 INFO [RS:1;2491923345a5:43689 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2491923345a5%2C43689%2C1733755621826, suffix=, logDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,43689,1733755621826, archiveDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/oldWALs, maxLogs=32 2024-12-09T14:47:03,452 INFO [RS:2;2491923345a5:44703 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2491923345a5%2C44703%2C1733755621870, suffix=, logDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,44703,1733755621870, archiveDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/oldWALs, maxLogs=32 2024-12-09T14:47:03,452 INFO [RS:0;2491923345a5:37129 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2491923345a5%2C37129%2C1733755621729, suffix=, logDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729, archiveDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/oldWALs, maxLogs=32 2024-12-09T14:47:03,469 DEBUG [RS:2;2491923345a5:44703 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,44703,1733755621870/2491923345a5%2C44703%2C1733755621870.1733755623457, exclude list is [], retry=0 2024-12-09T14:47:03,469 DEBUG [RS:0;2491923345a5:37129 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729/2491923345a5%2C37129%2C1733755621729.1733755623457, exclude list is [], retry=0 2024-12-09T14:47:03,470 DEBUG [RS:1;2491923345a5:43689 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,43689,1733755621826/2491923345a5%2C43689%2C1733755621826.1733755623457, exclude list is [], retry=0 2024-12-09T14:47:03,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:03,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:03,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:03,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:03,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:03,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:03,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:03,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:03,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:03,509 INFO [RS:1;2491923345a5:43689 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,43689,1733755621826/2491923345a5%2C43689%2C1733755621826.1733755623457 2024-12-09T14:47:03,509 DEBUG [RS:1;2491923345a5:43689 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429)] 2024-12-09T14:47:03,517 INFO [RS:2;2491923345a5:44703 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,44703,1733755621870/2491923345a5%2C44703%2C1733755621870.1733755623457 2024-12-09T14:47:03,517 DEBUG [RS:2;2491923345a5:44703 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:03,518 INFO [RS:0;2491923345a5:37129 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729/2491923345a5%2C37129%2C1733755621729.1733755623457 2024-12-09T14:47:03,519 DEBUG [RS:0;2491923345a5:37129 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:03,532 DEBUG [2491923345a5:40599 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-09T14:47:03,539 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(204): Hosts are {2491923345a5=0} racks are {/default-rack=0} 2024-12-09T14:47:03,545 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:47:03,545 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:47:03,545 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T14:47:03,545 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:47:03,545 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:47:03,545 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T14:47:03,545 INFO [2491923345a5:40599 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:47:03,545 INFO [2491923345a5:40599 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:47:03,545 INFO [2491923345a5:40599 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T14:47:03,545 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:47:03,551 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2491923345a5,43689,1733755621826 2024-12-09T14:47:03,557 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2491923345a5,43689,1733755621826, state=OPENING 2024-12-09T14:47:03,588 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T14:47:03,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:03,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:03,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:03,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:03,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:47:03,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:47:03,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:47:03,600 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:47:03,603 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T14:47:03,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2491923345a5,43689,1733755621826}] 2024-12-09T14:47:03,784 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:47:03,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38601, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:47:03,799 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T14:47:03,799 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:03,800 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T14:47:03,803 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2491923345a5%2C43689%2C1733755621826.meta, suffix=.meta, logDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,43689,1733755621826, archiveDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/oldWALs, maxLogs=32 2024-12-09T14:47:03,816 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,43689,1733755621826/2491923345a5%2C43689%2C1733755621826.meta.1733755623805.meta, exclude list is [], retry=0 2024-12-09T14:47:03,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:03,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:03,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:03,846 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,43689,1733755621826/2491923345a5%2C43689%2C1733755621826.meta.1733755623805.meta 2024-12-09T14:47:03,847 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:03,847 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:03,848 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T14:47:03,850 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T14:47:03,854 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T14:47:03,857 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T14:47:03,858 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:03,858 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T14:47:03,858 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T14:47:03,861 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T14:47:03,862 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T14:47:03,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:03,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:47:03,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T14:47:03,864 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T14:47:03,864 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:03,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:47:03,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T14:47:03,866 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T14:47:03,866 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:03,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:47:03,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T14:47:03,868 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T14:47:03,868 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:03,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T14:47:03,869 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T14:47:03,870 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740 2024-12-09T14:47:03,873 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740 2024-12-09T14:47:03,875 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T14:47:03,875 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T14:47:03,876 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T14:47:03,878 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T14:47:03,880 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66029056, jitterRate=-0.01609039306640625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T14:47:03,880 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T14:47:03,881 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733755623858Writing region info on filesystem at 1733755623858Initializing all the Stores at 1733755623860 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733755623860Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733755623860Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755623860Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733755623860Cleaning up temporary data from old regions at 1733755623875 (+15 ms)Running coprocessor post-open hooks at 1733755623880 (+5 ms)Region opened successfully at 1733755623881 (+1 ms) 2024-12-09T14:47:03,888 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733755623776 2024-12-09T14:47:03,898 DEBUG [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T14:47:03,898 INFO [RS_OPEN_META-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T14:47:03,900 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2491923345a5,43689,1733755621826 2024-12-09T14:47:03,902 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2491923345a5,43689,1733755621826, state=OPEN 2024-12-09T14:47:03,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:47:03,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:47:03,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:47:03,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T14:47:03,932 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:47:03,932 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:47:03,932 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:47:03,932 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T14:47:03,933 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2491923345a5,43689,1733755621826 2024-12-09T14:47:03,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T14:47:03,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2491923345a5,43689,1733755621826 in 328 msec 2024-12-09T14:47:03,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T14:47:03,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 826 msec 2024-12-09T14:47:03,951 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T14:47:03,951 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T14:47:03,967 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T14:47:03,968 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2491923345a5,43689,1733755621826, seqNum=-1] 2024-12-09T14:47:03,984 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:47:03,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36019, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:47:04,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0830 sec 2024-12-09T14:47:04,005 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733755624005, completionTime=-1 2024-12-09T14:47:04,008 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-09T14:47:04,008 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T14:47:04,030 INFO [master/2491923345a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-09T14:47:04,030 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733755684030 2024-12-09T14:47:04,030 INFO [master/2491923345a5:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733755744030 2024-12-09T14:47:04,031 INFO [master/2491923345a5:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-12-09T14:47:04,032 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T14:47:04,038 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,40599,1733755621019-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:04,038 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,40599,1733755621019-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:04,038 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,40599,1733755621019-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:04,039 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2491923345a5:40599, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:04,040 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:04,040 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:04,046 DEBUG [master/2491923345a5:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T14:47:04,070 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.095sec 2024-12-09T14:47:04,072 INFO [master/2491923345a5:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T14:47:04,073 INFO [master/2491923345a5:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T14:47:04,074 INFO [master/2491923345a5:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T14:47:04,075 INFO [master/2491923345a5:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T14:47:04,075 INFO [master/2491923345a5:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T14:47:04,076 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,40599,1733755621019-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T14:47:04,076 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,40599,1733755621019-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T14:47:04,081 DEBUG [master/2491923345a5:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T14:47:04,082 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T14:47:04,083 INFO [master/2491923345a5:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2491923345a5,40599,1733755621019-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:04,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6e3190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:47:04,112 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2491923345a5,40599,-1 for getting cluster id 2024-12-09T14:47:04,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T14:47:04,122 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20c7aab3-df44-4d3a-95f0-b2f8c0a2e453' 2024-12-09T14:47:04,125 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T14:47:04,125 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20c7aab3-df44-4d3a-95f0-b2f8c0a2e453" 2024-12-09T14:47:04,127 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161db4bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:47:04,127 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2491923345a5,40599,-1] 2024-12-09T14:47:04,129 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T14:47:04,131 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:47:04,132 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T14:47:04,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f808dbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T14:47:04,135 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T14:47:04,141 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2491923345a5,43689,1733755621826, seqNum=-1] 2024-12-09T14:47:04,141 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:47:04,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:47:04,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2491923345a5,40599,1733755621019 2024-12-09T14:47:04,164 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:35639/hbase 2024-12-09T14:47:04,177 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=366, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=8084 2024-12-09T14:47:04,194 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:04,197 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:04,198 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:04,202 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-58913208, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/hregion-58913208, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:04,216 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-58913208/hregion-58913208.1733755624204, exclude list is [], retry=0 2024-12-09T14:47:04,220 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:04,220 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:04,220 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:04,257 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-58913208/hregion-58913208.1733755624204 2024-12-09T14:47:04,258 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429)] 2024-12-09T14:47:04,258 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 98dd7746cb9a1c41348710b28c7f6dc0, NAME => 'testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/hbase 2024-12-09T14:47:04,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741838_1014 (size=64) 2024-12-09T14:47:04,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741838_1014 (size=64) 2024-12-09T14:47:04,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741838_1014 (size=64) 2024-12-09T14:47:04,271 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:04,273 INFO [StoreOpener-98dd7746cb9a1c41348710b28c7f6dc0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,275 INFO [StoreOpener-98dd7746cb9a1c41348710b28c7f6dc0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98dd7746cb9a1c41348710b28c7f6dc0 columnFamilyName a 2024-12-09T14:47:04,275 DEBUG [StoreOpener-98dd7746cb9a1c41348710b28c7f6dc0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:04,276 INFO [StoreOpener-98dd7746cb9a1c41348710b28c7f6dc0-1 {}] regionserver.HStore(327): Store=98dd7746cb9a1c41348710b28c7f6dc0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:04,277 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,278 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,278 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,279 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,279 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,282 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,286 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:04,287 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 98dd7746cb9a1c41348710b28c7f6dc0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64168978, jitterRate=-0.04380771517753601}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:47:04,288 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 98dd7746cb9a1c41348710b28c7f6dc0: Writing region info on filesystem at 1733755624272Initializing all the Stores at 1733755624273 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755624273Cleaning up temporary data from old regions at 1733755624279 (+6 ms)Region opened successfully at 1733755624288 (+9 ms) 2024-12-09T14:47:04,288 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 98dd7746cb9a1c41348710b28c7f6dc0, disabling compactions & flushes 2024-12-09T14:47:04,288 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0. 2024-12-09T14:47:04,288 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0. 2024-12-09T14:47:04,288 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0. after waiting 0 ms 2024-12-09T14:47:04,288 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0. 2024-12-09T14:47:04,289 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0. 2024-12-09T14:47:04,289 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 98dd7746cb9a1c41348710b28c7f6dc0: Waiting for close lock at 1733755624288Disabling compacts and flushes for region at 1733755624288Disabling writes for close at 1733755624288Writing region close event to WAL at 1733755624289 (+1 ms)Closed at 1733755624289 2024-12-09T14:47:04,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741837_1013 (size=175) 2024-12-09T14:47:04,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741837_1013 (size=175) 2024-12-09T14:47:04,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741837_1013 (size=175) 2024-12-09T14:47:04,301 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:04,301 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-58913208:(num 1733755624204) 2024-12-09T14:47:04,303 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T14:47:04,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741839_1015 (size=469) 2024-12-09T14:47:04,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741839_1015 (size=469) 2024-12-09T14:47:04,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741839_1015 (size=469) 2024-12-09T14:47:04,346 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T14:47:04,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741840_1016 (size=322) 2024-12-09T14:47:04,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741840_1016 (size=322) 2024-12-09T14:47:04,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741840_1016 (size=322) 2024-12-09T14:47:04,406 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-1, size=469 (469bytes) 2024-12-09T14:47:04,406 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-09T14:47:04,406 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-09T14:47:04,406 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-1 2024-12-09T14:47:04,411 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-1 after 3ms 2024-12-09T14:47:04,440 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:04,441 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-1 took 37ms 2024-12-09T14:47:04,446 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-1 so closing down 2024-12-09T14:47:04,446 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:04,449 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T14:47:04,472 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T14:47:04,473 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:04,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741841_1017 (size=469) 2024-12-09T14:47:04,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741841_1017 (size=469) 2024-12-09T14:47:04,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741841_1017 (size=469) 2024-12-09T14:47:04,485 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:04,488 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000002 2024-12-09T14:47:04,492 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-09T14:47:04,492 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-1, journal: Splitting hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-1, size=469 (469bytes) at 1733755624406Finishing writing output for hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-1 so closing down at 1733755624446 (+40 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000001-wal-1.temp at 1733755624473 (+27 ms)3 split writer threads finished at 1733755624473Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733755624485 (+12 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000002 at 1733755624488 (+3 ms)Processed 2 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733755624492 (+4 ms) 2024-12-09T14:47:04,505 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-2, size=322 (322bytes) 2024-12-09T14:47:04,505 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-2 2024-12-09T14:47:04,506 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-2 after 1ms 2024-12-09T14:47:04,531 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:04,531 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-2 took 26ms 2024-12-09T14:47:04,534 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-2 so closing down 2024-12-09T14:47:04,534 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:04,537 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T14:47:04,565 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T14:47:04,566 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:04,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741842_1018 (size=322) 2024-12-09T14:47:04,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741842_1018 (size=322) 2024-12-09T14:47:04,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741842_1018 (size=322) 2024-12-09T14:47:04,576 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:04,603 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:04,606 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-09T14:47:04,608 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-09T14:47:04,608 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-2, journal: Splitting hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-2, size=322 (322bytes) at 1733755624505Finishing writing output for hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-2 so closing down at 1733755624534 (+29 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000002-wal-2.temp at 1733755624565 (+31 ms)3 split writer threads finished at 1733755624566 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733755624576 (+10 ms)Processed 1 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733755624608 (+32 ms) 2024-12-09T14:47:04,608 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:04,610 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:04,623 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal.1733755624612, exclude list is [], retry=0 2024-12-09T14:47:04,627 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:04,628 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:04,629 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:04,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal.1733755624612 2024-12-09T14:47:04,653 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:04,653 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 98dd7746cb9a1c41348710b28c7f6dc0, NAME => 'testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:04,654 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:04,654 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,654 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,656 INFO [StoreOpener-98dd7746cb9a1c41348710b28c7f6dc0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,658 INFO [StoreOpener-98dd7746cb9a1c41348710b28c7f6dc0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98dd7746cb9a1c41348710b28c7f6dc0 columnFamilyName a 2024-12-09T14:47:04,658 DEBUG [StoreOpener-98dd7746cb9a1c41348710b28c7f6dc0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:04,659 INFO [StoreOpener-98dd7746cb9a1c41348710b28c7f6dc0-1 {}] regionserver.HStore(327): Store=98dd7746cb9a1c41348710b28c7f6dc0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:04,660 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,661 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,665 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,666 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000002 2024-12-09T14:47:04,693 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:04,698 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000002 2024-12-09T14:47:04,703 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 98dd7746cb9a1c41348710b28c7f6dc0 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T14:47:04,776 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/.tmp/a/683338955f81497fbe519f1e03d1b5e0 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733755624301/Put/seqid=0 2024-12-09T14:47:04,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741844_1020 (size=5170) 2024-12-09T14:47:04,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741844_1020 (size=5170) 2024-12-09T14:47:04,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741844_1020 (size=5170) 2024-12-09T14:47:04,799 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/.tmp/a/683338955f81497fbe519f1e03d1b5e0 2024-12-09T14:47:04,843 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/.tmp/a/683338955f81497fbe519f1e03d1b5e0 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/a/683338955f81497fbe519f1e03d1b5e0 2024-12-09T14:47:04,853 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/a/683338955f81497fbe519f1e03d1b5e0, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T14:47:04,858 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 98dd7746cb9a1c41348710b28c7f6dc0 in 156ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T14:47:04,860 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/0000000000000000002 2024-12-09T14:47:04,861 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,861 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,865 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 98dd7746cb9a1c41348710b28c7f6dc0 2024-12-09T14:47:04,868 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/98dd7746cb9a1c41348710b28c7f6dc0/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T14:47:04,869 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 98dd7746cb9a1c41348710b28c7f6dc0; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61627601, jitterRate=-0.08167718350887299}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:47:04,870 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 98dd7746cb9a1c41348710b28c7f6dc0: Writing region info on filesystem at 1733755624654Initializing all the Stores at 1733755624655 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755624655Obtaining lock to block concurrent updates at 1733755624703 (+48 ms)Preparing flush snapshotting stores in 98dd7746cb9a1c41348710b28c7f6dc0 at 1733755624703Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733755624706 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733755624194.98dd7746cb9a1c41348710b28c7f6dc0. at 1733755624706Flushing 98dd7746cb9a1c41348710b28c7f6dc0/a: creating writer at 1733755624708 (+2 ms)Flushing 98dd7746cb9a1c41348710b28c7f6dc0/a: appending metadata at 1733755624763 (+55 ms)Flushing 98dd7746cb9a1c41348710b28c7f6dc0/a: closing flushed file at 1733755624766 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59cb9245: reopening flushed file at 1733755624841 (+75 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 98dd7746cb9a1c41348710b28c7f6dc0 in 156ms, sequenceid=2, compaction requested=false; wal=null at 1733755624858 (+17 ms)Cleaning up temporary data from old regions at 1733755624861 (+3 ms)Region opened successfully at 1733755624870 (+9 ms) 2024-12-09T14:47:04,893 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=378 (was 366) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:35639/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:58780 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:49668 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:41502 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:58800 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:49836 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=229 (was 229), ProcessCount=11 (was 11), AvailableMemoryMB=7967 (was 8084) 2024-12-09T14:47:04,902 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=7967 2024-12-09T14:47:04,918 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:04,919 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:04,920 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:04,923 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-07763653, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/hregion-07763653, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:04,933 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-07763653/hregion-07763653.1733755624924, exclude list is [], retry=0 2024-12-09T14:47:04,937 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:04,937 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:04,937 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:04,960 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-07763653/hregion-07763653.1733755624924 2024-12-09T14:47:04,960 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429)] 2024-12-09T14:47:04,960 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b4a682071a06a8dec62cc2158e695aae, NAME => 'testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/hbase 2024-12-09T14:47:04,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741846_1022 (size=64) 2024-12-09T14:47:04,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741846_1022 (size=64) 2024-12-09T14:47:04,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741846_1022 (size=64) 2024-12-09T14:47:04,976 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:04,978 INFO [StoreOpener-b4a682071a06a8dec62cc2158e695aae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:04,981 INFO [StoreOpener-b4a682071a06a8dec62cc2158e695aae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4a682071a06a8dec62cc2158e695aae columnFamilyName a 2024-12-09T14:47:04,981 DEBUG [StoreOpener-b4a682071a06a8dec62cc2158e695aae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:04,981 INFO [StoreOpener-b4a682071a06a8dec62cc2158e695aae-1 {}] regionserver.HStore(327): Store=b4a682071a06a8dec62cc2158e695aae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:04,982 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:04,983 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:04,983 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:04,984 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:04,984 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:04,986 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:04,989 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:04,990 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b4a682071a06a8dec62cc2158e695aae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63143510, jitterRate=-0.05908837914466858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:47:04,990 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b4a682071a06a8dec62cc2158e695aae: Writing region info on filesystem at 1733755624976Initializing all the Stores at 1733755624977 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755624978 (+1 ms)Cleaning up temporary data from old regions at 1733755624984 (+6 ms)Region opened successfully at 1733755624990 (+6 ms) 2024-12-09T14:47:04,990 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b4a682071a06a8dec62cc2158e695aae, disabling compactions & flushes 2024-12-09T14:47:04,990 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae. 2024-12-09T14:47:04,990 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae. 2024-12-09T14:47:04,990 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae. after waiting 0 ms 2024-12-09T14:47:04,990 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae. 2024-12-09T14:47:04,991 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae. 2024-12-09T14:47:04,991 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b4a682071a06a8dec62cc2158e695aae: Waiting for close lock at 1733755624990Disabling compacts and flushes for region at 1733755624990Disabling writes for close at 1733755624990Writing region close event to WAL at 1733755624991 (+1 ms)Closed at 1733755624991 2024-12-09T14:47:04,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741845_1021 (size=175) 2024-12-09T14:47:04,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741845_1021 (size=175) 2024-12-09T14:47:04,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741845_1021 (size=175) 2024-12-09T14:47:04,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:04,997 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-07763653:(num 1733755624924) 2024-12-09T14:47:04,998 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T14:47:05,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741847_1023 (size=469) 2024-12-09T14:47:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741847_1023 (size=469) 2024-12-09T14:47:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741847_1023 (size=469) 2024-12-09T14:47:05,035 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-09T14:47:05,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741848_1024 (size=322) 2024-12-09T14:47:05,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741848_1024 (size=322) 2024-12-09T14:47:05,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741848_1024 (size=322) 2024-12-09T14:47:05,090 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-2, size=322 (322bytes) 2024-12-09T14:47:05,090 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-2 2024-12-09T14:47:05,091 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-2 after 1ms 2024-12-09T14:47:05,119 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:05,119 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-2 took 29ms 2024-12-09T14:47:05,122 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-2 so closing down 2024-12-09T14:47:05,122 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:05,124 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-09T14:47:05,147 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002-wal-2.temp 2024-12-09T14:47:05,147 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:05,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741849_1025 (size=322) 2024-12-09T14:47:05,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741849_1025 (size=322) 2024-12-09T14:47:05,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741849_1025 (size=322) 2024-12-09T14:47:05,155 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:05,157 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002 2024-12-09T14:47:05,157 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-09T14:47:05,157 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-2, journal: Splitting hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-2, size=322 (322bytes) at 1733755625090Finishing writing output for hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-2 so closing down at 1733755625122 (+32 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002-wal-2.temp at 1733755625147 (+25 ms)3 split writer threads finished at 1733755625148 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733755625155 (+7 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002 at 1733755625157 (+2 ms)Processed 1 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733755625157 2024-12-09T14:47:05,170 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-1, size=469 (469bytes) 2024-12-09T14:47:05,170 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-1 2024-12-09T14:47:05,171 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-1 after 1ms 2024-12-09T14:47:05,196 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:05,196 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-1 took 26ms 2024-12-09T14:47:05,199 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-1 so closing down 2024-12-09T14:47:05,199 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:05,201 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-09T14:47:05,224 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000001-wal-1.temp 2024-12-09T14:47:05,225 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:05,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741850_1026 (size=469) 2024-12-09T14:47:05,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741850_1026 (size=469) 2024-12-09T14:47:05,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741850_1026 (size=469) 2024-12-09T14:47:05,233 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:05,259 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:05,261 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002, length=322 2024-12-09T14:47:05,263 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002 2024-12-09T14:47:05,263 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-09T14:47:05,263 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-1, journal: Splitting hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-1, size=469 (469bytes) at 1733755625170Finishing writing output for hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-1 so closing down at 1733755625199 (+29 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000001-wal-1.temp at 1733755625224 (+25 ms)3 split writer threads finished at 1733755625225 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733755625233 (+8 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002 at 1733755625263 (+30 ms)Processed 2 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733755625263 2024-12-09T14:47:05,263 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:05,266 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:05,281 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal.1733755625266, exclude list is [], retry=0 2024-12-09T14:47:05,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:05,285 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:05,285 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:05,315 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal.1733755625266 2024-12-09T14:47:05,316 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:05,316 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b4a682071a06a8dec62cc2158e695aae, NAME => 'testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:05,316 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:05,316 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:05,316 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:05,318 INFO [StoreOpener-b4a682071a06a8dec62cc2158e695aae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:05,320 INFO [StoreOpener-b4a682071a06a8dec62cc2158e695aae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4a682071a06a8dec62cc2158e695aae columnFamilyName a 2024-12-09T14:47:05,320 DEBUG [StoreOpener-b4a682071a06a8dec62cc2158e695aae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:05,321 INFO [StoreOpener-b4a682071a06a8dec62cc2158e695aae-1 {}] regionserver.HStore(327): Store=b4a682071a06a8dec62cc2158e695aae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:05,321 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:05,322 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:05,324 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:05,325 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002 2024-12-09T14:47:05,350 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:05,352 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002 2024-12-09T14:47:05,352 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b4a682071a06a8dec62cc2158e695aae 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-09T14:47:05,370 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/.tmp/a/74248efbfd834c6fadf6397df37a2346 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733755624997/Put/seqid=0 2024-12-09T14:47:05,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741852_1028 (size=5170) 2024-12-09T14:47:05,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741852_1028 (size=5170) 2024-12-09T14:47:05,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741852_1028 (size=5170) 2024-12-09T14:47:05,379 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/.tmp/a/74248efbfd834c6fadf6397df37a2346 2024-12-09T14:47:05,386 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/.tmp/a/74248efbfd834c6fadf6397df37a2346 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/a/74248efbfd834c6fadf6397df37a2346 2024-12-09T14:47:05,395 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/a/74248efbfd834c6fadf6397df37a2346, entries=2, sequenceid=2, filesize=5.0 K 2024-12-09T14:47:05,396 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b4a682071a06a8dec62cc2158e695aae in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-12-09T14:47:05,397 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/0000000000000000002 2024-12-09T14:47:05,397 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:05,397 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:05,400 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b4a682071a06a8dec62cc2158e695aae 2024-12-09T14:47:05,403 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/b4a682071a06a8dec62cc2158e695aae/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-09T14:47:05,405 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b4a682071a06a8dec62cc2158e695aae; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74125277, jitterRate=0.10455270111560822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:47:05,405 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b4a682071a06a8dec62cc2158e695aae: Writing region info on filesystem at 1733755625316Initializing all the Stores at 1733755625317 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755625317Obtaining lock to block concurrent updates at 1733755625352 (+35 ms)Preparing flush snapshotting stores in b4a682071a06a8dec62cc2158e695aae at 1733755625352Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733755625352Flushing stores of testReplayEditsWrittenIntoWAL,,1733755624918.b4a682071a06a8dec62cc2158e695aae. at 1733755625352Flushing b4a682071a06a8dec62cc2158e695aae/a: creating writer at 1733755625353 (+1 ms)Flushing b4a682071a06a8dec62cc2158e695aae/a: appending metadata at 1733755625369 (+16 ms)Flushing b4a682071a06a8dec62cc2158e695aae/a: closing flushed file at 1733755625369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65c41fe5: reopening flushed file at 1733755625385 (+16 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for b4a682071a06a8dec62cc2158e695aae in 44ms, sequenceid=2, compaction requested=false; wal=null at 1733755625396 (+11 ms)Cleaning up temporary data from old regions at 1733755625398 (+2 ms)Region opened successfully at 1733755625405 (+7 ms) 2024-12-09T14:47:05,420 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=388 (was 378) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:58868 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:41350 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:41584 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:58780 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:49668 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:49890 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=229 (was 229), ProcessCount=11 (was 11), AvailableMemoryMB=7958 (was 7967) 2024-12-09T14:47:05,430 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=388, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=7957 2024-12-09T14:47:05,443 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:05,446 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:05,446 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:05,449 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-60380751, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/hregion-60380751, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:05,461 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-60380751/hregion-60380751.1733755625450, exclude list is [], retry=0 2024-12-09T14:47:05,464 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:05,465 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:05,465 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:05,487 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-60380751/hregion-60380751.1733755625450 2024-12-09T14:47:05,488 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:05,488 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ceddc8b1045fd662e60109422611cc96, NAME => 'testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/hbase 2024-12-09T14:47:05,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741854_1030 (size=64) 2024-12-09T14:47:05,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741854_1030 (size=64) 2024-12-09T14:47:05,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741854_1030 (size=64) 2024-12-09T14:47:05,502 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:05,504 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:05,505 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceddc8b1045fd662e60109422611cc96 columnFamilyName a 2024-12-09T14:47:05,506 DEBUG [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:05,506 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(327): Store=ceddc8b1045fd662e60109422611cc96/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:05,506 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:05,509 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceddc8b1045fd662e60109422611cc96 columnFamilyName b 2024-12-09T14:47:05,509 DEBUG [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:05,509 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(327): Store=ceddc8b1045fd662e60109422611cc96/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:05,510 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:05,511 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceddc8b1045fd662e60109422611cc96 columnFamilyName c 2024-12-09T14:47:05,512 DEBUG [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:05,512 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(327): Store=ceddc8b1045fd662e60109422611cc96/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:05,512 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:05,513 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:05,514 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:05,515 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:05,515 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:05,516 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:05,517 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:05,520 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:05,520 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ceddc8b1045fd662e60109422611cc96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68067233, jitterRate=0.014280810952186584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:05,521 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ceddc8b1045fd662e60109422611cc96: Writing region info on filesystem at 1733755625502Initializing all the Stores at 1733755625503 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755625503Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755625503Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755625503Cleaning up temporary data from old regions at 1733755625515 (+12 ms)Region opened successfully at 1733755625520 (+5 ms) 2024-12-09T14:47:05,521 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ceddc8b1045fd662e60109422611cc96, disabling compactions & flushes 2024-12-09T14:47:05,521 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. 2024-12-09T14:47:05,521 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. 2024-12-09T14:47:05,521 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. after waiting 0 ms 2024-12-09T14:47:05,521 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. 2024-12-09T14:47:05,521 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. 2024-12-09T14:47:05,522 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ceddc8b1045fd662e60109422611cc96: Waiting for close lock at 1733755625521Disabling compacts and flushes for region at 1733755625521Disabling writes for close at 1733755625521Writing region close event to WAL at 1733755625521Closed at 1733755625521 2024-12-09T14:47:05,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741853_1029 (size=175) 2024-12-09T14:47:05,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741853_1029 (size=175) 2024-12-09T14:47:05,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741853_1029 (size=175) 2024-12-09T14:47:05,528 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:05,528 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-60380751:(num 1733755625450) 2024-12-09T14:47:05,528 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:05,530 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:05,541 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530, exclude list is [], retry=0 2024-12-09T14:47:05,544 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:05,545 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:05,545 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:05,575 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 2024-12-09T14:47:05,575 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:05,757 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530, size=0 (0bytes) 2024-12-09T14:47:05,758 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 might be still open, length is 0 2024-12-09T14:47:05,758 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 2024-12-09T14:47:05,759 WARN [IPC Server handler 1 on default port 35639 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-09T14:47:05,759 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 after 1ms 2024-12-09T14:47:06,194 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:41620 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37163:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41620 dst: /127.0.0.1:37163 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37163 remote=/127.0.0.1:41620]. Total timeout mills is 60000, 59536 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:06,195 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:49912 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33317:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49912 dst: /127.0.0.1:33317 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:06,195 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:58898 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46775:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58898 dst: /127.0.0.1:46775 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:06,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741855_1032 (size=497775) 2024-12-09T14:47:06,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741855_1032 (size=497775) 2024-12-09T14:47:06,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741855_1032 (size=497775) 2024-12-09T14:47:09,373 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T14:47:09,418 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T14:47:09,761 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 after 4002ms 2024-12-09T14:47:09,790 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:09,792 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 took 4035ms 2024-12-09T14:47:09,796 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733755625530.temp 2024-12-09T14:47:09,820 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000000001-wal.1733755625530.temp 2024-12-09T14:47:09,892 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530; continuing. 2024-12-09T14:47:09,892 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 so closing down 2024-12-09T14:47:09,892 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:09,893 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:09,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741856_1033 (size=497783) 2024-12-09T14:47:09,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741856_1033 (size=497783) 2024-12-09T14:47:09,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741856_1033 (size=497783) 2024-12-09T14:47:09,897 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000000001-wal.1733755625530.temp (wrote 3002 edits, skipped 0 edits in 39 ms) 2024-12-09T14:47:09,898 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000000001-wal.1733755625530.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000003002 2024-12-09T14:47:09,898 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 106 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T14:47:09,899 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530, journal: Splitting hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530, size=0 (0bytes) at 1733755625758Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000000001-wal.1733755625530.temp at 1733755629820 (+4062 ms)Split 1024 edits, skipped 0 edits. at 1733755629849 (+29 ms)Split 2048 edits, skipped 0 edits. at 1733755629873 (+24 ms)Finishing writing output for hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 so closing down at 1733755629892 (+19 ms)3 split writer threads finished at 1733755629893 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000000001-wal.1733755625530.temp (wrote 3002 edits, skipped 0 edits in 39 ms) at 1733755629897 (+4 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000000001-wal.1733755625530.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000003002 at 1733755629898 (+1 ms)Processed 3002 edits across 1 Regions in 106 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530, size=0, length=0, corrupted=false, cancelled=false at 1733755629899 (+1 ms) 2024-12-09T14:47:09,901 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 to hdfs://localhost:35639/hbase/oldWALs/wal.1733755625530 2024-12-09T14:47:09,902 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000003002 2024-12-09T14:47:09,902 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:09,904 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:09,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755629905, exclude list is [], retry=0 2024-12-09T14:47:09,919 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:09,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:09,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:09,944 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755629905 2024-12-09T14:47:09,944 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429)] 2024-12-09T14:47:09,944 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:09,946 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:09,947 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceddc8b1045fd662e60109422611cc96 columnFamilyName a 2024-12-09T14:47:09,947 DEBUG [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:09,948 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(327): Store=ceddc8b1045fd662e60109422611cc96/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:09,948 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:09,949 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceddc8b1045fd662e60109422611cc96 columnFamilyName b 2024-12-09T14:47:09,949 DEBUG [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:09,950 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(327): Store=ceddc8b1045fd662e60109422611cc96/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:09,950 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:09,951 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceddc8b1045fd662e60109422611cc96 columnFamilyName c 2024-12-09T14:47:09,951 DEBUG [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:09,952 INFO [StoreOpener-ceddc8b1045fd662e60109422611cc96-1 {}] regionserver.HStore(327): Store=ceddc8b1045fd662e60109422611cc96/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:09,952 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:09,953 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:09,955 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:09,956 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000003002 2024-12-09T14:47:09,990 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:10,020 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T14:47:10,326 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ceddc8b1045fd662e60109422611cc96 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T14:47:10,367 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/a/fa49a3e47abf4892a0d2cbf083dd4cb9 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733755625583/Put/seqid=0 2024-12-09T14:47:10,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741858_1035 (size=50463) 2024-12-09T14:47:10,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741858_1035 (size=50463) 2024-12-09T14:47:10,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741858_1035 (size=50463) 2024-12-09T14:47:10,378 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/a/fa49a3e47abf4892a0d2cbf083dd4cb9 2024-12-09T14:47:10,386 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/a/fa49a3e47abf4892a0d2cbf083dd4cb9 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/a/fa49a3e47abf4892a0d2cbf083dd4cb9 2024-12-09T14:47:10,395 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/a/fa49a3e47abf4892a0d2cbf083dd4cb9, entries=754, sequenceid=754, filesize=49.3 K 2024-12-09T14:47:10,395 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for ceddc8b1045fd662e60109422611cc96 in 69ms, sequenceid=754, compaction requested=false; wal=null 2024-12-09T14:47:10,407 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T14:47:10,407 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ceddc8b1045fd662e60109422611cc96 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T14:47:10,416 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/a/d89ada87577843dca8a73e2201b5a2a6 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733755625619/Put/seqid=0 2024-12-09T14:47:10,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741859_1036 (size=20072) 2024-12-09T14:47:10,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741859_1036 (size=20072) 2024-12-09T14:47:10,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741859_1036 (size=20072) 2024-12-09T14:47:10,426 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/a/d89ada87577843dca8a73e2201b5a2a6 2024-12-09T14:47:10,453 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/b/31f9efaab22b42838c2ec67b29041052 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733755625640/Put/seqid=0 2024-12-09T14:47:10,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741860_1037 (size=35835) 2024-12-09T14:47:10,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741860_1037 (size=35835) 2024-12-09T14:47:10,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741860_1037 (size=35835) 2024-12-09T14:47:10,461 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/b/31f9efaab22b42838c2ec67b29041052 2024-12-09T14:47:10,469 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/a/d89ada87577843dca8a73e2201b5a2a6 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/a/d89ada87577843dca8a73e2201b5a2a6 2024-12-09T14:47:10,476 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/a/d89ada87577843dca8a73e2201b5a2a6, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-09T14:47:10,477 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/b/31f9efaab22b42838c2ec67b29041052 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/b/31f9efaab22b42838c2ec67b29041052 2024-12-09T14:47:10,483 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/b/31f9efaab22b42838c2ec67b29041052, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-09T14:47:10,483 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ceddc8b1045fd662e60109422611cc96 in 76ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-09T14:47:10,499 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T14:47:10,500 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ceddc8b1045fd662e60109422611cc96 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-09T14:47:10,508 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/b/a1d40e166c994889b2b8355ce1201119 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733755625663/Put/seqid=0 2024-12-09T14:47:10,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741861_1038 (size=35082) 2024-12-09T14:47:10,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741861_1038 (size=35082) 2024-12-09T14:47:10,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741861_1038 (size=35082) 2024-12-09T14:47:10,519 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/b/a1d40e166c994889b2b8355ce1201119 2024-12-09T14:47:10,542 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/c/89f99c7cd0624a16809006bab5117dc5 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733755625696/Put/seqid=0 2024-12-09T14:47:10,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741862_1039 (size=20825) 2024-12-09T14:47:10,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741862_1039 (size=20825) 2024-12-09T14:47:10,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741862_1039 (size=20825) 2024-12-09T14:47:10,551 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/c/89f99c7cd0624a16809006bab5117dc5 2024-12-09T14:47:10,558 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/b/a1d40e166c994889b2b8355ce1201119 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/b/a1d40e166c994889b2b8355ce1201119 2024-12-09T14:47:10,565 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/b/a1d40e166c994889b2b8355ce1201119, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-09T14:47:10,566 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/c/89f99c7cd0624a16809006bab5117dc5 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/c/89f99c7cd0624a16809006bab5117dc5 2024-12-09T14:47:10,574 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/c/89f99c7cd0624a16809006bab5117dc5, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-09T14:47:10,574 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ceddc8b1045fd662e60109422611cc96 in 74ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-09T14:47:10,585 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733755625728/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. 2024-12-09T14:47:10,589 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000003002 2024-12-09T14:47:10,589 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-09T14:47:10,589 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ceddc8b1045fd662e60109422611cc96 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-09T14:47:10,600 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/c/3cbfa2683c2b4e2697ccc1436502183e is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733755625702/Put/seqid=0 2024-12-09T14:47:10,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741863_1040 (size=50301) 2024-12-09T14:47:10,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741863_1040 (size=50301) 2024-12-09T14:47:10,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741863_1040 (size=50301) 2024-12-09T14:47:10,610 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/c/3cbfa2683c2b4e2697ccc1436502183e 2024-12-09T14:47:10,617 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3cbfa2683c2b4e2697ccc1436502183e 2024-12-09T14:47:10,619 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/.tmp/c/3cbfa2683c2b4e2697ccc1436502183e as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/c/3cbfa2683c2b4e2697ccc1436502183e 2024-12-09T14:47:10,625 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3cbfa2683c2b4e2697ccc1436502183e 2024-12-09T14:47:10,626 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/c/3cbfa2683c2b4e2697ccc1436502183e, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-09T14:47:10,626 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for ceddc8b1045fd662e60109422611cc96 in 37ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-09T14:47:10,627 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/0000000000000003002 2024-12-09T14:47:10,628 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:10,628 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:10,629 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T14:47:10,630 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ceddc8b1045fd662e60109422611cc96 2024-12-09T14:47:10,633 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenIntoWAL/ceddc8b1045fd662e60109422611cc96/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-09T14:47:10,634 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ceddc8b1045fd662e60109422611cc96; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65739283, jitterRate=-0.020408347249031067}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T14:47:10,634 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ceddc8b1045fd662e60109422611cc96: Writing region info on filesystem at 1733755629944Initializing all the Stores at 1733755629946 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755629946Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755629946Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755629946Cleaning up temporary data from old regions at 1733755630628 (+682 ms)Region opened successfully at 1733755630634 (+6 ms) 2024-12-09T14:47:10,696 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ceddc8b1045fd662e60109422611cc96, disabling compactions & flushes 2024-12-09T14:47:10,696 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. 2024-12-09T14:47:10,696 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. 2024-12-09T14:47:10,697 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. after waiting 0 ms 2024-12-09T14:47:10,697 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. 2024-12-09T14:47:10,703 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733755625444.ceddc8b1045fd662e60109422611cc96. 2024-12-09T14:47:10,703 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ceddc8b1045fd662e60109422611cc96: Waiting for close lock at 1733755630696Disabling compacts and flushes for region at 1733755630696Disabling writes for close at 1733755630697 (+1 ms)Writing region close event to WAL at 1733755630703 (+6 ms)Closed at 1733755630703 2024-12-09T14:47:10,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741857_1034 (size=175) 2024-12-09T14:47:10,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741857_1034 (size=175) 2024-12-09T14:47:10,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741857_1034 (size=175) 2024-12-09T14:47:10,709 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:10,709 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733755629905) 2024-12-09T14:47:10,725 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=402 (was 388) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:42081 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35639 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2049457240_22 at /127.0.0.1:49952 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2049457240_22 at /127.0.0.1:58946 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:35639 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:46141 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2049457240_22 at /127.0.0.1:41676 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46141 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2049457240_22 at /127.0.0.1:58954 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42081 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=227 (was 229), ProcessCount=11 (was 11), AvailableMemoryMB=7880 (was 7957) 2024-12-09T14:47:10,737 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=402, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=227, ProcessCount=11, AvailableMemoryMB=7876 2024-12-09T14:47:10,752 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:10,754 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:10,755 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:10,758 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-78632388, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/hregion-78632388, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:10,771 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-78632388/hregion-78632388.1733755630759, exclude list is [], retry=0 2024-12-09T14:47:10,775 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:10,775 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:10,776 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:10,803 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-78632388/hregion-78632388.1733755630759 2024-12-09T14:47:10,804 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:10,804 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8f6641c9272d8629b553664f3582b6fc, NAME => 'test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/hbase 2024-12-09T14:47:10,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741865_1042 (size=43) 2024-12-09T14:47:10,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741865_1042 (size=43) 2024-12-09T14:47:10,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741865_1042 (size=43) 2024-12-09T14:47:10,818 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:10,820 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:10,821 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f6641c9272d8629b553664f3582b6fc columnFamilyName a 2024-12-09T14:47:10,821 DEBUG [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:10,822 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(327): Store=8f6641c9272d8629b553664f3582b6fc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:10,822 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:10,823 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f6641c9272d8629b553664f3582b6fc columnFamilyName b 2024-12-09T14:47:10,824 DEBUG [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:10,824 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(327): Store=8f6641c9272d8629b553664f3582b6fc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:10,824 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:10,826 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f6641c9272d8629b553664f3582b6fc columnFamilyName c 2024-12-09T14:47:10,826 DEBUG [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:10,826 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(327): Store=8f6641c9272d8629b553664f3582b6fc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:10,827 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:10,827 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:10,828 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:10,829 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:10,829 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:10,829 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:10,831 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:10,834 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:10,835 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f6641c9272d8629b553664f3582b6fc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71783497, jitterRate=0.06965745985507965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:10,836 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f6641c9272d8629b553664f3582b6fc: Writing region info on filesystem at 1733755630818Initializing all the Stores at 1733755630819 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755630819Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755630820 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755630820Cleaning up temporary data from old regions at 1733755630829 (+9 ms)Region opened successfully at 1733755630836 (+7 ms) 2024-12-09T14:47:10,837 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8f6641c9272d8629b553664f3582b6fc, disabling compactions & flushes 2024-12-09T14:47:10,837 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. 2024-12-09T14:47:10,837 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. 2024-12-09T14:47:10,837 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. after waiting 0 ms 2024-12-09T14:47:10,837 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. 2024-12-09T14:47:10,837 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. 2024-12-09T14:47:10,837 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8f6641c9272d8629b553664f3582b6fc: Waiting for close lock at 1733755630837Disabling compacts and flushes for region at 1733755630837Disabling writes for close at 1733755630837Writing region close event to WAL at 1733755630837Closed at 1733755630837 2024-12-09T14:47:10,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741864_1041 (size=175) 2024-12-09T14:47:10,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741864_1041 (size=175) 2024-12-09T14:47:10,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741864_1041 (size=175) 2024-12-09T14:47:10,844 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:10,844 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-78632388:(num 1733755630759) 2024-12-09T14:47:10,844 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:10,846 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:10,858 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847, exclude list is [], retry=0 2024-12-09T14:47:10,861 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:10,861 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:10,862 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:10,883 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847 2024-12-09T14:47:10,883 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:11,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741866_1043 (size=371388) 2024-12-09T14:47:11,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741866_1043 (size=371388) 2024-12-09T14:47:11,044 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847 not finished, retry = 0 2024-12-09T14:47:11,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741866_1043 (size=371388) 2024-12-09T14:47:11,170 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847, size=362.7 K (371388bytes) 2024-12-09T14:47:11,170 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847 2024-12-09T14:47:11,170 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847 after 0ms 2024-12-09T14:47:11,207 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:11,209 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847 took 40ms 2024-12-09T14:47:11,213 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733755630847.temp 2024-12-09T14:47:11,247 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847 so closing down 2024-12-09T14:47:11,247 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:11,250 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000000001-wal.1733755630847.temp 2024-12-09T14:47:11,266 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:11,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741867_1044 (size=371388) 2024-12-09T14:47:11,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741867_1044 (size=371388) 2024-12-09T14:47:11,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741867_1044 (size=371388) 2024-12-09T14:47:11,274 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000000001-wal.1733755630847.temp (wrote 3000 edits, skipped 0 edits in 15 ms) 2024-12-09T14:47:11,276 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000000001-wal.1733755630847.temp to hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003000 2024-12-09T14:47:11,276 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 67 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-09T14:47:11,276 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847, journal: Splitting hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847, size=362.7 K (371388bytes) at 1733755631170Split 1024 edits, skipped 0 edits. at 1733755631223 (+53 ms)Split 2048 edits, skipped 0 edits. at 1733755631236 (+13 ms)Finishing writing output for hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847 so closing down at 1733755631247 (+11 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000000001-wal.1733755630847.temp at 1733755631250 (+3 ms)3 split writer threads finished at 1733755631266 (+16 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000000001-wal.1733755630847.temp (wrote 3000 edits, skipped 0 edits in 15 ms) at 1733755631274 (+8 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000000001-wal.1733755630847.temp to hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003000 at 1733755631276 (+2 ms)Processed 3000 edits across 1 Regions in 67 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733755631276 2024-12-09T14:47:11,278 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755630847 to hdfs://localhost:35639/hbase/oldWALs/wal.1733755630847 2024-12-09T14:47:11,279 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003000 2024-12-09T14:47:11,279 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:11,281 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:11,293 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282, exclude list is [], retry=0 2024-12-09T14:47:11,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:11,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:11,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:11,324 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282 2024-12-09T14:47:11,324 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:11,336 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T14:47:11,336 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T14:47:11,338 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T14:47:11,338 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T14:47:11,339 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T14:47:11,339 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T14:47:11,340 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-09T14:47:11,340 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-09T14:47:11,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741868_1045 (size=371515) 2024-12-09T14:47:11,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741868_1045 (size=371515) 2024-12-09T14:47:11,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741868_1045 (size=371515) 2024-12-09T14:47:11,473 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282, size=362.8 K (371515bytes) 2024-12-09T14:47:11,473 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282 2024-12-09T14:47:11,473 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282 after 0ms 2024-12-09T14:47:11,501 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:11,503 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282 took 31ms 2024-12-09T14:47:11,507 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733755631282.temp 2024-12-09T14:47:11,541 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282 so closing down 2024-12-09T14:47:11,542 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:11,548 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003001-wal.1733755631282.temp 2024-12-09T14:47:11,564 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:11,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741869_1046 (size=371515) 2024-12-09T14:47:11,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741869_1046 (size=371515) 2024-12-09T14:47:11,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741869_1046 (size=371515) 2024-12-09T14:47:11,572 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003001-wal.1733755631282.temp (wrote 3000 edits, skipped 0 edits in 15 ms) 2024-12-09T14:47:11,574 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003001-wal.1733755631282.temp to hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000006000 2024-12-09T14:47:11,574 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-09T14:47:11,574 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282, journal: Splitting hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282, size=362.8 K (371515bytes) at 1733755631473Split 1024 edits, skipped 0 edits. at 1733755631517 (+44 ms)Split 2048 edits, skipped 0 edits. at 1733755631530 (+13 ms)Finishing writing output for hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282 so closing down at 1733755631541 (+11 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003001-wal.1733755631282.temp at 1733755631548 (+7 ms)3 split writer threads finished at 1733755631564 (+16 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003001-wal.1733755631282.temp (wrote 3000 edits, skipped 0 edits in 15 ms) at 1733755631572 (+8 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003001-wal.1733755631282.temp to hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000006000 at 1733755631574 (+2 ms)Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733755631574 2024-12-09T14:47:11,577 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631282 to hdfs://localhost:35639/hbase/oldWALs/wal.1733755631282 2024-12-09T14:47:11,578 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000006000 2024-12-09T14:47:11,578 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:11,580 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/test2727-manual,16010,1733755630751, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:11,600 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631581, exclude list is [], retry=0 2024-12-09T14:47:11,603 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:11,604 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:11,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:11,633 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733755630751/wal.1733755631581 2024-12-09T14:47:11,633 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:11,633 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f6641c9272d8629b553664f3582b6fc, NAME => 'test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:11,634 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:11,634 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,634 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,636 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,637 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f6641c9272d8629b553664f3582b6fc columnFamilyName a 2024-12-09T14:47:11,637 DEBUG [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:11,638 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(327): Store=8f6641c9272d8629b553664f3582b6fc/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:11,638 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,639 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f6641c9272d8629b553664f3582b6fc columnFamilyName b 2024-12-09T14:47:11,639 DEBUG [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:11,640 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(327): Store=8f6641c9272d8629b553664f3582b6fc/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:11,640 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,642 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f6641c9272d8629b553664f3582b6fc columnFamilyName c 2024-12-09T14:47:11,642 DEBUG [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:11,642 INFO [StoreOpener-8f6641c9272d8629b553664f3582b6fc-1 {}] regionserver.HStore(327): Store=8f6641c9272d8629b553664f3582b6fc/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:11,643 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,644 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,646 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,648 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003000 2024-12-09T14:47:11,678 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:11,709 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003000 2024-12-09T14:47:11,711 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000006000 2024-12-09T14:47:11,740 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:11,793 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000006000 2024-12-09T14:47:11,793 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8f6641c9272d8629b553664f3582b6fc 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-09T14:47:11,819 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/.tmp/a/63552619920c45c6ae236f6e043567a0 is 41, key is test2727/a:100/1733755631328/Put/seqid=0 2024-12-09T14:47:11,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741871_1048 (size=84227) 2024-12-09T14:47:11,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741871_1048 (size=84227) 2024-12-09T14:47:11,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741871_1048 (size=84227) 2024-12-09T14:47:11,828 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/.tmp/a/63552619920c45c6ae236f6e043567a0 2024-12-09T14:47:11,861 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/.tmp/b/c1646604872f41eab18486d5b6d8445d is 41, key is test2727/b:100/1733755631369/Put/seqid=0 2024-12-09T14:47:11,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741872_1049 (size=84609) 2024-12-09T14:47:11,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741872_1049 (size=84609) 2024-12-09T14:47:11,867 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/.tmp/b/c1646604872f41eab18486d5b6d8445d 2024-12-09T14:47:11,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741872_1049 (size=84609) 2024-12-09T14:47:11,890 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/.tmp/c/c677c91a5f2a443fa44f218ff165ab90 is 41, key is test2727/c:100/1733755631413/Put/seqid=0 2024-12-09T14:47:11,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741873_1050 (size=84609) 2024-12-09T14:47:11,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741873_1050 (size=84609) 2024-12-09T14:47:11,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741873_1050 (size=84609) 2024-12-09T14:47:11,897 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/.tmp/c/c677c91a5f2a443fa44f218ff165ab90 2024-12-09T14:47:11,903 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/.tmp/a/63552619920c45c6ae236f6e043567a0 as hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/a/63552619920c45c6ae236f6e043567a0 2024-12-09T14:47:11,909 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/a/63552619920c45c6ae236f6e043567a0, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-09T14:47:11,911 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/.tmp/b/c1646604872f41eab18486d5b6d8445d as hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/b/c1646604872f41eab18486d5b6d8445d 2024-12-09T14:47:11,917 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/b/c1646604872f41eab18486d5b6d8445d, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T14:47:11,918 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/.tmp/c/c677c91a5f2a443fa44f218ff165ab90 as hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/c/c677c91a5f2a443fa44f218ff165ab90 2024-12-09T14:47:11,924 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/c/c677c91a5f2a443fa44f218ff165ab90, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-09T14:47:11,925 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 8f6641c9272d8629b553664f3582b6fc in 131ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-09T14:47:11,926 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000003000 2024-12-09T14:47:11,926 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/0000000000000006000 2024-12-09T14:47:11,927 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,927 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,928 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:11,930 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f6641c9272d8629b553664f3582b6fc 2024-12-09T14:47:11,932 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/test2727/8f6641c9272d8629b553664f3582b6fc/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-09T14:47:11,933 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f6641c9272d8629b553664f3582b6fc; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66786905, jitterRate=-0.0047975629568099976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:11,934 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f6641c9272d8629b553664f3582b6fc: Writing region info on filesystem at 1733755631634Initializing all the Stores at 1733755631635 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755631635Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755631636 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755631636Obtaining lock to block concurrent updates at 1733755631793 (+157 ms)Preparing flush snapshotting stores in 8f6641c9272d8629b553664f3582b6fc at 1733755631793Finished memstore snapshotting test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733755631793Flushing stores of test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. at 1733755631793Flushing 8f6641c9272d8629b553664f3582b6fc/a: creating writer at 1733755631793Flushing 8f6641c9272d8629b553664f3582b6fc/a: appending metadata at 1733755631818 (+25 ms)Flushing 8f6641c9272d8629b553664f3582b6fc/a: closing flushed file at 1733755631818Flushing 8f6641c9272d8629b553664f3582b6fc/b: creating writer at 1733755631835 (+17 ms)Flushing 8f6641c9272d8629b553664f3582b6fc/b: appending metadata at 1733755631859 (+24 ms)Flushing 8f6641c9272d8629b553664f3582b6fc/b: closing flushed file at 1733755631859Flushing 8f6641c9272d8629b553664f3582b6fc/c: creating writer at 1733755631873 (+14 ms)Flushing 8f6641c9272d8629b553664f3582b6fc/c: appending metadata at 1733755631890 (+17 ms)Flushing 8f6641c9272d8629b553664f3582b6fc/c: closing flushed file at 1733755631890Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@525e21b6: reopening flushed file at 1733755631902 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71fcb4a: reopening flushed file at 1733755631909 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@affdeeb: reopening flushed file at 1733755631917 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 8f6641c9272d8629b553664f3582b6fc in 131ms, sequenceid=6000, compaction requested=false; wal=null at 1733755631925 (+8 ms)Cleaning up temporary data from old regions at 1733755631927 (+2 ms)Region opened successfully at 1733755631934 (+7 ms) 2024-12-09T14:47:11,935 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-09T14:47:11,935 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8f6641c9272d8629b553664f3582b6fc, disabling compactions & flushes 2024-12-09T14:47:11,935 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. 2024-12-09T14:47:11,935 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. 2024-12-09T14:47:11,935 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. after waiting 0 ms 2024-12-09T14:47:11,935 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. 2024-12-09T14:47:11,937 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733755630753.8f6641c9272d8629b553664f3582b6fc. 2024-12-09T14:47:11,937 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8f6641c9272d8629b553664f3582b6fc: Waiting for close lock at 1733755631935Disabling compacts and flushes for region at 1733755631935Disabling writes for close at 1733755631935Writing region close event to WAL at 1733755631937 (+2 ms)Closed at 1733755631937 2024-12-09T14:47:11,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741870_1047 (size=175) 2024-12-09T14:47:11,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741870_1047 (size=175) 2024-12-09T14:47:11,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741870_1047 (size=175) 2024-12-09T14:47:11,943 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:11,943 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733755631581) 2024-12-09T14:47:11,955 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=404 (was 402) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:35380 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:49952 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:51646 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=227 (was 227), ProcessCount=11 (was 11), AvailableMemoryMB=7663 (was 7876) 2024-12-09T14:47:11,965 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=227, ProcessCount=11, AvailableMemoryMB=7662 2024-12-09T14:47:11,978 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:11,985 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:11,985 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733755631985 2024-12-09T14:47:12,011 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985 2024-12-09T14:47:12,012 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429)] 2024-12-09T14:47:12,013 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => aac8b5c3ec489dcf67a359ed945eef9c, NAME => 'testSequentialEditLogSeqNum,,1733755631979.aac8b5c3ec489dcf67a359ed945eef9c.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:12,014 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733755631979.aac8b5c3ec489dcf67a359ed945eef9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:12,014 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for aac8b5c3ec489dcf67a359ed945eef9c 2024-12-09T14:47:12,014 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for aac8b5c3ec489dcf67a359ed945eef9c 2024-12-09T14:47:12,015 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c doesn't exist for region: aac8b5c3ec489dcf67a359ed945eef9c on table testSequentialEditLogSeqNum 2024-12-09T14:47:12,015 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: aac8b5c3ec489dcf67a359ed945eef9c on table testSequentialEditLogSeqNum 2024-12-09T14:47:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741875_1052 (size=62) 2024-12-09T14:47:12,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741875_1052 (size=62) 2024-12-09T14:47:12,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741875_1052 (size=62) 2024-12-09T14:47:12,030 INFO [StoreOpener-aac8b5c3ec489dcf67a359ed945eef9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region aac8b5c3ec489dcf67a359ed945eef9c 2024-12-09T14:47:12,032 INFO [StoreOpener-aac8b5c3ec489dcf67a359ed945eef9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aac8b5c3ec489dcf67a359ed945eef9c columnFamilyName a 2024-12-09T14:47:12,032 DEBUG [StoreOpener-aac8b5c3ec489dcf67a359ed945eef9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:12,033 INFO [StoreOpener-aac8b5c3ec489dcf67a359ed945eef9c-1 {}] regionserver.HStore(327): Store=aac8b5c3ec489dcf67a359ed945eef9c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:12,033 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aac8b5c3ec489dcf67a359ed945eef9c 2024-12-09T14:47:12,034 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c 2024-12-09T14:47:12,034 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c 2024-12-09T14:47:12,035 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aac8b5c3ec489dcf67a359ed945eef9c 2024-12-09T14:47:12,035 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aac8b5c3ec489dcf67a359ed945eef9c 2024-12-09T14:47:12,037 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aac8b5c3ec489dcf67a359ed945eef9c 2024-12-09T14:47:12,039 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:12,040 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aac8b5c3ec489dcf67a359ed945eef9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70599083, jitterRate=0.05200831592082977}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:47:12,040 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aac8b5c3ec489dcf67a359ed945eef9c: Writing region info on filesystem at 1733755632014Initializing all the Stores at 1733755632029 (+15 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755632029Cleaning up temporary data from old regions at 1733755632035 (+6 ms)Region opened successfully at 1733755632040 (+5 ms) 2024-12-09T14:47:12,050 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing aac8b5c3ec489dcf67a359ed945eef9c 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-09T14:47:12,070 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/.tmp/a/842b5007202e41f1865ec2aec7e667bc is 81, key is testSequentialEditLogSeqNum/a:x0/1733755632040/Put/seqid=0 2024-12-09T14:47:12,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741876_1053 (size=5833) 2024-12-09T14:47:12,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741876_1053 (size=5833) 2024-12-09T14:47:12,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741876_1053 (size=5833) 2024-12-09T14:47:12,078 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/.tmp/a/842b5007202e41f1865ec2aec7e667bc 2024-12-09T14:47:12,086 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/.tmp/a/842b5007202e41f1865ec2aec7e667bc as hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/a/842b5007202e41f1865ec2aec7e667bc 2024-12-09T14:47:12,093 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/a/842b5007202e41f1865ec2aec7e667bc, entries=10, sequenceid=13, filesize=5.7 K 2024-12-09T14:47:12,094 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for aac8b5c3ec489dcf67a359ed945eef9c in 44ms, sequenceid=13, compaction requested=false 2024-12-09T14:47:12,094 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for aac8b5c3ec489dcf67a359ed945eef9c: 2024-12-09T14:47:12,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T14:47:12,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T14:47:12,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T14:47:12,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T14:47:12,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-09T14:47:12,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741874_1051 (size=3256) 2024-12-09T14:47:12,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741874_1051 (size=3256) 2024-12-09T14:47:12,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741874_1051 (size=3256) 2024-12-09T14:47:12,116 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985, size=3.2 K (3256bytes) 2024-12-09T14:47:12,116 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985 2024-12-09T14:47:12,117 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985 after 0ms 2024-12-09T14:47:12,139 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:12,139 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985 took 23ms 2024-12-09T14:47:12,142 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985 so closing down 2024-12-09T14:47:12,142 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:12,143 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733755631985.temp 2024-12-09T14:47:12,179 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/recovered.edits/0000000000000000003-wal.1733755631985.temp 2024-12-09T14:47:12,180 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:12,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741877_1054 (size=2650) 2024-12-09T14:47:12,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741877_1054 (size=2650) 2024-12-09T14:47:12,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741877_1054 (size=2650) 2024-12-09T14:47:12,188 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/recovered.edits/0000000000000000003-wal.1733755631985.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:12,189 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/recovered.edits/0000000000000000003-wal.1733755631985.temp to hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/recovered.edits/0000000000000000020 2024-12-09T14:47:12,190 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 50 ms; skipped=2; WAL=hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-09T14:47:12,190 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985, journal: Splitting hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985, size=3.2 K (3256bytes) at 1733755632116Finishing writing output for hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985 so closing down at 1733755632142 (+26 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/recovered.edits/0000000000000000003-wal.1733755631985.temp at 1733755632179 (+37 ms)3 split writer threads finished at 1733755632180 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/recovered.edits/0000000000000000003-wal.1733755631985.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733755632188 (+8 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/recovered.edits/0000000000000000003-wal.1733755631985.temp to hdfs://localhost:35639/hbase/data/default/testSequentialEditLogSeqNum/aac8b5c3ec489dcf67a359ed945eef9c/recovered.edits/0000000000000000020 at 1733755632189 (+1 ms)Processed 17 edits across 1 Regions in 50 ms; skipped=2; WAL=hdfs://localhost:35639/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733755631977/wal.1733755631985, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733755632190 (+1 ms) 2024-12-09T14:47:12,202 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=411 (was 404) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:35380 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:51646 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=227 (was 227), ProcessCount=11 (was 11), AvailableMemoryMB=7659 (was 7662) 2024-12-09T14:47:12,213 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=227, ProcessCount=11, AvailableMemoryMB=7659 2024-12-09T14:47:12,227 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:12,229 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:12,263 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:12,267 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26738695, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/hregion-26738695, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:12,280 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26738695/hregion-26738695.1733755632267, exclude list is [], retry=0 2024-12-09T14:47:12,283 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:12,283 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:12,283 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:12,319 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26738695/hregion-26738695.1733755632267 2024-12-09T14:47:12,319 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:12,319 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2c9247be0a013cbb37d45919fe855416, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/hbase 2024-12-09T14:47:12,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741879_1056 (size=70) 2024-12-09T14:47:12,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741879_1056 (size=70) 2024-12-09T14:47:12,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741879_1056 (size=70) 2024-12-09T14:47:12,330 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:12,331 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,333 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9247be0a013cbb37d45919fe855416 columnFamilyName a 2024-12-09T14:47:12,333 DEBUG [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:12,334 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(327): Store=2c9247be0a013cbb37d45919fe855416/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:12,334 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,335 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9247be0a013cbb37d45919fe855416 columnFamilyName b 2024-12-09T14:47:12,336 DEBUG [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:12,336 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(327): Store=2c9247be0a013cbb37d45919fe855416/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:12,337 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,338 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9247be0a013cbb37d45919fe855416 columnFamilyName c 2024-12-09T14:47:12,338 DEBUG [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:12,339 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(327): Store=2c9247be0a013cbb37d45919fe855416/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:12,339 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,340 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,340 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,342 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,342 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,342 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:12,343 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,346 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:12,346 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2c9247be0a013cbb37d45919fe855416; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70143503, jitterRate=0.04521964490413666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:12,347 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2c9247be0a013cbb37d45919fe855416: Writing region info on filesystem at 1733755632330Initializing all the Stores at 1733755632331 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755632331Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755632331Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755632331Cleaning up temporary data from old regions at 1733755632342 (+11 ms)Region opened successfully at 1733755632347 (+5 ms) 2024-12-09T14:47:12,348 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2c9247be0a013cbb37d45919fe855416, disabling compactions & flushes 2024-12-09T14:47:12,348 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. 2024-12-09T14:47:12,348 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. 2024-12-09T14:47:12,348 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. after waiting 0 ms 2024-12-09T14:47:12,348 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. 2024-12-09T14:47:12,348 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. 2024-12-09T14:47:12,349 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2c9247be0a013cbb37d45919fe855416: Waiting for close lock at 1733755632348Disabling compacts and flushes for region at 1733755632348Disabling writes for close at 1733755632348Writing region close event to WAL at 1733755632348Closed at 1733755632348 2024-12-09T14:47:12,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741878_1055 (size=175) 2024-12-09T14:47:12,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741878_1055 (size=175) 2024-12-09T14:47:12,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741878_1055 (size=175) 2024-12-09T14:47:12,355 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:12,356 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26738695:(num 1733755632267) 2024-12-09T14:47:12,356 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:12,358 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:12,370 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358, exclude list is [], retry=0 2024-12-09T14:47:12,373 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:12,374 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:12,374 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:12,407 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 2024-12-09T14:47:12,407 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429)] 2024-12-09T14:47:12,407 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c9247be0a013cbb37d45919fe855416, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:12,408 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:12,408 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,408 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,410 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,411 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9247be0a013cbb37d45919fe855416 columnFamilyName a 2024-12-09T14:47:12,411 DEBUG [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:12,412 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(327): Store=2c9247be0a013cbb37d45919fe855416/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:12,412 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,413 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9247be0a013cbb37d45919fe855416 columnFamilyName b 2024-12-09T14:47:12,413 DEBUG [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:12,413 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(327): Store=2c9247be0a013cbb37d45919fe855416/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:12,414 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,414 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9247be0a013cbb37d45919fe855416 columnFamilyName c 2024-12-09T14:47:12,414 DEBUG [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:12,415 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(327): Store=2c9247be0a013cbb37d45919fe855416/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:12,415 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,416 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,417 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,418 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,418 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,418 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:12,420 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:12,421 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2c9247be0a013cbb37d45919fe855416; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64574031, jitterRate=-0.037771955132484436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:12,422 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2c9247be0a013cbb37d45919fe855416: Writing region info on filesystem at 1733755632408Initializing all the Stores at 1733755632409 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755632409Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755632410 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755632410Cleaning up temporary data from old regions at 1733755632418 (+8 ms)Region opened successfully at 1733755632421 (+3 ms) 2024-12-09T14:47:12,426 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733755632425/Put/seqid=0 2024-12-09T14:47:12,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741881_1058 (size=4826) 2024-12-09T14:47:12,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741881_1058 (size=4826) 2024-12-09T14:47:12,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741881_1058 (size=4826) 2024-12-09T14:47:12,433 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35639/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 2c9247be0a013cbb37d45919fe855416/a 2024-12-09T14:47:12,440 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-09T14:47:12,440 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T14:47:12,441 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2c9247be0a013cbb37d45919fe855416: 2024-12-09T14:47:12,442 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/a/b3f3b0d6423b4d4b8a50eadd3c969864_SeqId_3_ 2024-12-09T14:47:12,443 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35639/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 2c9247be0a013cbb37d45919fe855416/a as hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/a/b3f3b0d6423b4d4b8a50eadd3c969864_SeqId_3_ - updating store file list. 2024-12-09T14:47:12,448 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for b3f3b0d6423b4d4b8a50eadd3c969864_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T14:47:12,449 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/a/b3f3b0d6423b4d4b8a50eadd3c969864_SeqId_3_ into 2c9247be0a013cbb37d45919fe855416/a 2024-12-09T14:47:12,449 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35639/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 2c9247be0a013cbb37d45919fe855416/a (new location: hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/a/b3f3b0d6423b4d4b8a50eadd3c969864_SeqId_3_) 2024-12-09T14:47:12,486 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358, size=0 (0bytes) 2024-12-09T14:47:12,486 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 might be still open, length is 0 2024-12-09T14:47:12,486 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 2024-12-09T14:47:12,487 WARN [IPC Server handler 4 on default port 35639 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-09T14:47:12,487 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 after 1ms 2024-12-09T14:47:15,186 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:53600 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33317:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53600 dst: /127.0.0.1:33317 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33317 remote=/127.0.0.1:53600]. Total timeout mills is 60000, 57269 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:15,187 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:51736 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37163:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51736 dst: /127.0.0.1:37163 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:15,187 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:35460 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46775:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35460 dst: /127.0.0.1:46775 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:15,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741880_1059 (size=606) 2024-12-09T14:47:15,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741880_1059 (size=606) 2024-12-09T14:47:16,488 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 after 4002ms 2024-12-09T14:47:16,517 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:16,517 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 took 4031ms 2024-12-09T14:47:16,520 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358; continuing. 2024-12-09T14:47:16,520 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 so closing down 2024-12-09T14:47:16,520 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:16,521 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733755632358.temp 2024-12-09T14:47:16,544 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005-wal.1733755632358.temp 2024-12-09T14:47:16,544 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:16,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741882_1060 (size=329) 2024-12-09T14:47:16,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741882_1060 (size=329) 2024-12-09T14:47:16,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741882_1060 (size=329) 2024-12-09T14:47:16,550 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005-wal.1733755632358.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:16,551 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005-wal.1733755632358.temp to hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005 2024-12-09T14:47:16,552 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 34 ms; skipped=1; WAL=hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T14:47:16,552 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358, journal: Splitting hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358, size=0 (0bytes) at 1733755632486Finishing writing output for hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 so closing down at 1733755636520 (+4034 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005-wal.1733755632358.temp at 1733755636544 (+24 ms)3 split writer threads finished at 1733755636544Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005-wal.1733755632358.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733755636550 (+6 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005-wal.1733755632358.temp to hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005 at 1733755636551 (+1 ms)Processed 2 edits across 1 Regions in 34 ms; skipped=1; WAL=hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358, size=0, length=0, corrupted=false, cancelled=false at 1733755636552 (+1 ms) 2024-12-09T14:47:16,553 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 to hdfs://localhost:35639/hbase/oldWALs/wal.1733755632358 2024-12-09T14:47:16,554 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005 2024-12-09T14:47:16,554 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:16,556 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:16,570 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755636557, exclude list is [], retry=0 2024-12-09T14:47:16,573 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:16,573 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:16,573 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:16,596 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755636557 2024-12-09T14:47:16,596 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:16,597 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c9247be0a013cbb37d45919fe855416, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:16,597 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:16,597 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,597 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,598 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,599 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9247be0a013cbb37d45919fe855416 columnFamilyName a 2024-12-09T14:47:16,599 DEBUG [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:16,605 DEBUG [StoreFileOpener-2c9247be0a013cbb37d45919fe855416-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for b3f3b0d6423b4d4b8a50eadd3c969864_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-09T14:47:16,605 DEBUG [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/a/b3f3b0d6423b4d4b8a50eadd3c969864_SeqId_3_ 2024-12-09T14:47:16,606 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(327): Store=2c9247be0a013cbb37d45919fe855416/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:16,606 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,607 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9247be0a013cbb37d45919fe855416 columnFamilyName b 2024-12-09T14:47:16,607 DEBUG [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:16,607 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(327): Store=2c9247be0a013cbb37d45919fe855416/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:16,607 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,608 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9247be0a013cbb37d45919fe855416 columnFamilyName c 2024-12-09T14:47:16,608 DEBUG [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:16,608 INFO [StoreOpener-2c9247be0a013cbb37d45919fe855416-1 {}] regionserver.HStore(327): Store=2c9247be0a013cbb37d45919fe855416/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:16,609 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,609 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,611 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,611 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005 2024-12-09T14:47:16,632 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:16,633 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005 2024-12-09T14:47:16,633 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2c9247be0a013cbb37d45919fe855416 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-09T14:47:16,646 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/.tmp/a/124521233f94428ba6e4aa566a663269 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733755632454/Put/seqid=0 2024-12-09T14:47:16,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741884_1062 (size=5149) 2024-12-09T14:47:16,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741884_1062 (size=5149) 2024-12-09T14:47:16,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741884_1062 (size=5149) 2024-12-09T14:47:16,652 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/.tmp/a/124521233f94428ba6e4aa566a663269 2024-12-09T14:47:16,656 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/.tmp/a/124521233f94428ba6e4aa566a663269 as hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/a/124521233f94428ba6e4aa566a663269 2024-12-09T14:47:16,662 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/a/124521233f94428ba6e4aa566a663269, entries=1, sequenceid=5, filesize=5.0 K 2024-12-09T14:47:16,662 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 2c9247be0a013cbb37d45919fe855416 in 29ms, sequenceid=5, compaction requested=false; wal=null 2024-12-09T14:47:16,663 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/0000000000000000005 2024-12-09T14:47:16,664 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,664 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,665 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:16,666 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2c9247be0a013cbb37d45919fe855416 2024-12-09T14:47:16,668 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c9247be0a013cbb37d45919fe855416/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-09T14:47:16,669 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2c9247be0a013cbb37d45919fe855416; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65947477, jitterRate=-0.017306014895439148}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:16,669 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2c9247be0a013cbb37d45919fe855416: Writing region info on filesystem at 1733755636597Initializing all the Stores at 1733755636598 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755636598Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755636598Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755636598Obtaining lock to block concurrent updates at 1733755636633 (+35 ms)Preparing flush snapshotting stores in 2c9247be0a013cbb37d45919fe855416 at 1733755636633Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733755636633Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. at 1733755636633Flushing 2c9247be0a013cbb37d45919fe855416/a: creating writer at 1733755636633Flushing 2c9247be0a013cbb37d45919fe855416/a: appending metadata at 1733755636645 (+12 ms)Flushing 2c9247be0a013cbb37d45919fe855416/a: closing flushed file at 1733755636645Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d618a6c: reopening flushed file at 1733755636656 (+11 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 2c9247be0a013cbb37d45919fe855416 in 29ms, sequenceid=5, compaction requested=false; wal=null at 1733755636662 (+6 ms)Cleaning up temporary data from old regions at 1733755636664 (+2 ms)Region opened successfully at 1733755636669 (+5 ms) 2024-12-09T14:47:16,673 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2c9247be0a013cbb37d45919fe855416, disabling compactions & flushes 2024-12-09T14:47:16,673 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. 2024-12-09T14:47:16,673 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. 2024-12-09T14:47:16,673 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. after waiting 0 ms 2024-12-09T14:47:16,673 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. 2024-12-09T14:47:16,674 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733755632228.2c9247be0a013cbb37d45919fe855416. 2024-12-09T14:47:16,674 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2c9247be0a013cbb37d45919fe855416: Waiting for close lock at 1733755636673Disabling compacts and flushes for region at 1733755636673Disabling writes for close at 1733755636673Writing region close event to WAL at 1733755636674 (+1 ms)Closed at 1733755636674 2024-12-09T14:47:16,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741883_1061 (size=175) 2024-12-09T14:47:16,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741883_1061 (size=175) 2024-12-09T14:47:16,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741883_1061 (size=175) 2024-12-09T14:47:16,678 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:16,679 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733755636557) 2024-12-09T14:47:16,691 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413 (was 411) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1066955226_22 at /127.0.0.1:51754 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:35639 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35639 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1066955226_22 at /127.0.0.1:35508 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=209 (was 227), ProcessCount=11 (was 11), AvailableMemoryMB=7657 (was 7659) 2024-12-09T14:47:16,703 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=413, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=209, ProcessCount=11, AvailableMemoryMB=7656 2024-12-09T14:47:16,717 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:16,721 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T14:47:16,725 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2491923345a5,40599,1733755621019 2024-12-09T14:47:16,727 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33e3ab50 2024-12-09T14:47:16,727 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T14:47:16,729 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T14:47:16,732 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T14:47:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T14:47:16,741 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T14:47:16,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-09T14:47:16,744 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:16,746 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T14:47:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:47:16,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741885_1063 (size=694) 2024-12-09T14:47:16,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741885_1063 (size=694) 2024-12-09T14:47:16,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741885_1063 (size=694) 2024-12-09T14:47:16,758 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 10975ad07e6cf31ab36e8c23dc0846f3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f 2024-12-09T14:47:16,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741886_1064 (size=77) 2024-12-09T14:47:16,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741886_1064 (size=77) 2024-12-09T14:47:16,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741886_1064 (size=77) 2024-12-09T14:47:16,768 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:16,768 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 10975ad07e6cf31ab36e8c23dc0846f3, disabling compactions & flushes 2024-12-09T14:47:16,768 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:16,768 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:16,768 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. after waiting 0 ms 2024-12-09T14:47:16,768 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:16,768 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:16,768 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 10975ad07e6cf31ab36e8c23dc0846f3: Waiting for close lock at 1733755636768Disabling compacts and flushes for region at 1733755636768Disabling writes for close at 1733755636768Writing region close event to WAL at 1733755636768Closed at 1733755636768 2024-12-09T14:47:16,770 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T14:47:16,774 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733755636770"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733755636770"}]},"ts":"1733755636770"} 2024-12-09T14:47:16,777 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T14:47:16,778 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T14:47:16,780 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733755636778"}]},"ts":"1733755636778"} 2024-12-09T14:47:16,783 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-09T14:47:16,784 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2491923345a5=0} racks are {/default-rack=0} 2024-12-09T14:47:16,785 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:47:16,785 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:47:16,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-09T14:47:16,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:47:16,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:47:16,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-09T14:47:16,786 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:47:16,786 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:47:16,786 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-09T14:47:16,786 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:47:16,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, ASSIGN}] 2024-12-09T14:47:16,790 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, ASSIGN 2024-12-09T14:47:16,792 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, ASSIGN; state=OFFLINE, location=2491923345a5,37129,1733755621729; forceNewPlan=false, retain=false 2024-12-09T14:47:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:47:16,946 INFO [2491923345a5:40599 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T14:47:16,947 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=OPENING, regionLocation=2491923345a5,37129,1733755621729 2024-12-09T14:47:16,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, ASSIGN because future has completed 2024-12-09T14:47:16,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,37129,1733755621729}] 2024-12-09T14:47:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:47:17,106 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:47:17,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45361, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:47:17,112 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,112 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 10975ad07e6cf31ab36e8c23dc0846f3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:17,112 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,113 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:17,113 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,113 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,114 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,116 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf1 2024-12-09T14:47:17,116 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:17,116 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:17,117 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,118 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf2 2024-12-09T14:47:17,118 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:17,119 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:17,119 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,120 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,120 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,121 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,121 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,122 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T14:47:17,124 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,126 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:17,127 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 10975ad07e6cf31ab36e8c23dc0846f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70118404, jitterRate=0.044845640659332275}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T14:47:17,127 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,127 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 10975ad07e6cf31ab36e8c23dc0846f3: Running coprocessor pre-open hook at 1733755637113Writing region info on filesystem at 1733755637113Initializing all the Stores at 1733755637114 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755637114Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755637114Cleaning up temporary data from old regions at 1733755637121 (+7 ms)Running coprocessor post-open hooks at 1733755637127 (+6 ms)Region opened successfully at 1733755637127 2024-12-09T14:47:17,129 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., pid=6, masterSystemTime=1733755637105 2024-12-09T14:47:17,132 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,132 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,133 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=OPEN, openSeqNum=2, regionLocation=2491923345a5,37129,1733755621729 2024-12-09T14:47:17,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,37129,1733755621729 because future has completed 2024-12-09T14:47:17,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T14:47:17,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,37129,1733755621729 in 186 msec 2024-12-09T14:47:17,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T14:47:17,144 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, ASSIGN in 353 msec 2024-12-09T14:47:17,145 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T14:47:17,146 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733755637145"}]},"ts":"1733755637145"} 2024-12-09T14:47:17,148 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-09T14:47:17,149 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T14:47:17,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 415 msec 2024-12-09T14:47:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T14:47:17,382 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-09T14:47:17,382 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-09T14:47:17,385 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T14:47:17,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-09T14:47:17,393 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T14:47:17,394 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-09T14:47:17,406 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=2] 2024-12-09T14:47:17,408 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:47:17,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:47:17,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=10975ad07e6cf31ab36e8c23dc0846f3, source=2491923345a5,37129,1733755621729, destination=2491923345a5,43689,1733755621826, warming up region on 2491923345a5,43689,1733755621826 2024-12-09T14:47:17,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=10975ad07e6cf31ab36e8c23dc0846f3, source=2491923345a5,37129,1733755621729, destination=2491923345a5,43689,1733755621826, running balancer 2024-12-09T14:47:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE 2024-12-09T14:47:17,429 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE 2024-12-09T14:47:17,432 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=CLOSING, regionLocation=2491923345a5,37129,1733755621729 2024-12-09T14:47:17,432 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] regionserver.HRegion(7855): Warmup {ENCODED => 10975ad07e6cf31ab36e8c23dc0846f3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:17,434 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE because future has completed 2024-12-09T14:47:17,435 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf1 2024-12-09T14:47:17,435 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:17,436 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:17,436 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,437 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T14:47:17,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,37129,1733755621729}] 2024-12-09T14:47:17,437 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf2 2024-12-09T14:47:17,437 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:17,438 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] regionserver.HRegion(1722): Closing 10975ad07e6cf31ab36e8c23dc0846f3, disabling compactions & flushes 2024-12-09T14:47:17,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. after waiting 0 ms 2024-12-09T14:47:17,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,441 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] regionserver.HRegion(1676): Region close journal for 10975ad07e6cf31ab36e8c23dc0846f3: Waiting for close lock at 1733755637438Disabling compacts and flushes for region at 1733755637438Disabling writes for close at 1733755637438Writing region close event to WAL at 1733755637440 (+2 ms)Closed at 1733755637440 2024-12-09T14:47:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-09T14:47:17,596 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,596 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T14:47:17,596 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 10975ad07e6cf31ab36e8c23dc0846f3, disabling compactions & flushes 2024-12-09T14:47:17,596 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,596 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,597 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. after waiting 0 ms 2024-12-09T14:47:17,597 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,597 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 10975ad07e6cf31ab36e8c23dc0846f3 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-09T14:47:17,611 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf1/eef99cada17240548893e51d5c018ca2 is 35, key is r1/cf1:q/1733755637411/Put/seqid=0 2024-12-09T14:47:17,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741887_1065 (size=4783) 2024-12-09T14:47:17,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741887_1065 (size=4783) 2024-12-09T14:47:17,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741887_1065 (size=4783) 2024-12-09T14:47:17,619 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf1/eef99cada17240548893e51d5c018ca2 2024-12-09T14:47:17,626 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf1/eef99cada17240548893e51d5c018ca2 as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/eef99cada17240548893e51d5c018ca2 2024-12-09T14:47:17,635 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/eef99cada17240548893e51d5c018ca2, entries=1, sequenceid=5, filesize=4.7 K 2024-12-09T14:47:17,636 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 10975ad07e6cf31ab36e8c23dc0846f3 in 39ms, sequenceid=5, compaction requested=false 2024-12-09T14:47:17,636 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-09T14:47:17,642 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T14:47:17,644 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,644 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 10975ad07e6cf31ab36e8c23dc0846f3: Waiting for close lock at 1733755637596Running coprocessor pre-close hooks at 1733755637596Disabling compacts and flushes for region at 1733755637596Disabling writes for close at 1733755637597 (+1 ms)Obtaining lock to block concurrent updates at 1733755637597Preparing flush snapshotting stores in 10975ad07e6cf31ab36e8c23dc0846f3 at 1733755637597Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733755637597Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. at 1733755637598 (+1 ms)Flushing 10975ad07e6cf31ab36e8c23dc0846f3/cf1: creating writer at 1733755637598Flushing 10975ad07e6cf31ab36e8c23dc0846f3/cf1: appending metadata at 1733755637611 (+13 ms)Flushing 10975ad07e6cf31ab36e8c23dc0846f3/cf1: closing flushed file at 1733755637611Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@de91e20: reopening flushed file at 1733755637625 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 10975ad07e6cf31ab36e8c23dc0846f3 in 39ms, sequenceid=5, compaction requested=false at 1733755637636 (+11 ms)Writing region close event to WAL at 1733755637639 (+3 ms)Running coprocessor post-close hooks at 1733755637643 (+4 ms)Closed at 1733755637644 (+1 ms) 2024-12-09T14:47:17,645 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 10975ad07e6cf31ab36e8c23dc0846f3 move to 2491923345a5,43689,1733755621826 record at close sequenceid=5 2024-12-09T14:47:17,647 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,648 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=CLOSED 2024-12-09T14:47:17,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,37129,1733755621729 because future has completed 2024-12-09T14:47:17,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T14:47:17,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,37129,1733755621729 in 216 msec 2024-12-09T14:47:17,658 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE; state=CLOSED, location=2491923345a5,43689,1733755621826; forceNewPlan=false, retain=false 2024-12-09T14:47:17,809 INFO [2491923345a5:40599 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T14:47:17,810 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=OPENING, regionLocation=2491923345a5,43689,1733755621826 2024-12-09T14:47:17,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE because future has completed 2024-12-09T14:47:17,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,43689,1733755621826}] 2024-12-09T14:47:17,976 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:17,976 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 10975ad07e6cf31ab36e8c23dc0846f3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:17,977 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,977 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:17,977 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,977 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,979 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,981 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf1 2024-12-09T14:47:17,981 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:17,989 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/eef99cada17240548893e51d5c018ca2 2024-12-09T14:47:17,990 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:17,990 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,991 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf2 2024-12-09T14:47:17,991 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:17,991 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:17,992 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,992 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,994 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,994 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,994 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,995 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T14:47:17,997 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,997 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 10975ad07e6cf31ab36e8c23dc0846f3; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61173421, jitterRate=-0.08844499289989471}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T14:47:17,997 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:17,998 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 10975ad07e6cf31ab36e8c23dc0846f3: Running coprocessor pre-open hook at 1733755637977Writing region info on filesystem at 1733755637977Initializing all the Stores at 1733755637979 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755637979Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755637979Cleaning up temporary data from old regions at 1733755637995 (+16 ms)Running coprocessor post-open hooks at 1733755637997 (+2 ms)Region opened successfully at 1733755637998 (+1 ms) 2024-12-09T14:47:17,999 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., pid=9, masterSystemTime=1733755637971 2024-12-09T14:47:18,002 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,002 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=OPEN, openSeqNum=9, regionLocation=2491923345a5,43689,1733755621826 2024-12-09T14:47:18,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,43689,1733755621826 because future has completed 2024-12-09T14:47:18,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-09T14:47:18,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,43689,1733755621826 in 191 msec 2024-12-09T14:47:18,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE in 584 msec 2024-12-09T14:47:18,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:47:18,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:47:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:46184 deadline: 1733755698042, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2491923345a5 port=43689 startCode=1733755621826. As of locationSeqNum=5. 2024-12-09T14:47:18,051 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2491923345a5 port=43689 startCode=1733755621826. As of locationSeqNum=5. 2024-12-09T14:47:18,052 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2491923345a5 port=43689 startCode=1733755621826. As of locationSeqNum=5. 2024-12-09T14:47:18,052 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,43689,1733755621826, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2491923345a5 port=43689 startCode=1733755621826. As of locationSeqNum=5. 2024-12-09T14:47:18,169 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 10975ad07e6cf31ab36e8c23dc0846f3 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-09T14:47:18,184 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf1/457ed6d84a2c4548b2f735fa7cf7c875 is 29, key is r1/cf1:/1733755638160/DeleteFamily/seqid=0 2024-12-09T14:47:18,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741888_1066 (size=4906) 2024-12-09T14:47:18,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741888_1066 (size=4906) 2024-12-09T14:47:18,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741888_1066 (size=4906) 2024-12-09T14:47:18,193 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf1/457ed6d84a2c4548b2f735fa7cf7c875 2024-12-09T14:47:18,199 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 457ed6d84a2c4548b2f735fa7cf7c875 2024-12-09T14:47:18,217 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf2/520ecfa3ee8f4e04a4566db4e286a106 is 29, key is r1/cf2:/1733755638160/DeleteFamily/seqid=0 2024-12-09T14:47:18,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741889_1067 (size=4906) 2024-12-09T14:47:18,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741889_1067 (size=4906) 2024-12-09T14:47:18,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741889_1067 (size=4906) 2024-12-09T14:47:18,224 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf2/520ecfa3ee8f4e04a4566db4e286a106 2024-12-09T14:47:18,231 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 520ecfa3ee8f4e04a4566db4e286a106 2024-12-09T14:47:18,232 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf1/457ed6d84a2c4548b2f735fa7cf7c875 as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/457ed6d84a2c4548b2f735fa7cf7c875 2024-12-09T14:47:18,237 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 457ed6d84a2c4548b2f735fa7cf7c875 2024-12-09T14:47:18,237 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/457ed6d84a2c4548b2f735fa7cf7c875, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T14:47:18,238 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf2/520ecfa3ee8f4e04a4566db4e286a106 as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/520ecfa3ee8f4e04a4566db4e286a106 2024-12-09T14:47:18,245 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 520ecfa3ee8f4e04a4566db4e286a106 2024-12-09T14:47:18,245 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/520ecfa3ee8f4e04a4566db4e286a106, entries=1, sequenceid=12, filesize=4.8 K 2024-12-09T14:47:18,246 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 10975ad07e6cf31ab36e8c23dc0846f3 in 77ms, sequenceid=12, compaction requested=false 2024-12-09T14:47:18,246 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 10975ad07e6cf31ab36e8c23dc0846f3: 2024-12-09T14:47:18,248 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T14:47:18,250 DEBUG [Time-limited test {}] regionserver.HStore(1541): 10975ad07e6cf31ab36e8c23dc0846f3/cf1 is initiating major compaction (all files) 2024-12-09T14:47:18,250 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:47:18,250 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:18,250 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 10975ad07e6cf31ab36e8c23dc0846f3/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,250 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/eef99cada17240548893e51d5c018ca2, hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/457ed6d84a2c4548b2f735fa7cf7c875] into tmpdir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp, totalSize=9.5 K 2024-12-09T14:47:18,251 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting eef99cada17240548893e51d5c018ca2, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733755637411 2024-12-09T14:47:18,252 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 457ed6d84a2c4548b2f735fa7cf7c875, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T14:47:18,262 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 10975ad07e6cf31ab36e8c23dc0846f3#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T14:47:18,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741890_1068 (size=4626) 2024-12-09T14:47:18,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741890_1068 (size=4626) 2024-12-09T14:47:18,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741890_1068 (size=4626) 2024-12-09T14:47:18,277 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf1/21d1c01f806e42349817e88535f61c50 as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/21d1c01f806e42349817e88535f61c50 2024-12-09T14:47:18,291 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 10975ad07e6cf31ab36e8c23dc0846f3/cf1 of 10975ad07e6cf31ab36e8c23dc0846f3 into 21d1c01f806e42349817e88535f61c50(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T14:47:18,292 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 10975ad07e6cf31ab36e8c23dc0846f3: 2024-12-09T14:47:18,292 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-09T14:47:18,292 DEBUG [Time-limited test {}] regionserver.HStore(1541): 10975ad07e6cf31ab36e8c23dc0846f3/cf2 is initiating major compaction (all files) 2024-12-09T14:47:18,292 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T14:47:18,292 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T14:47:18,292 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 10975ad07e6cf31ab36e8c23dc0846f3/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,292 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/520ecfa3ee8f4e04a4566db4e286a106] into tmpdir=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp, totalSize=4.8 K 2024-12-09T14:47:18,293 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 520ecfa3ee8f4e04a4566db4e286a106, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-09T14:47:18,299 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 10975ad07e6cf31ab36e8c23dc0846f3#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T14:47:18,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741891_1069 (size=4592) 2024-12-09T14:47:18,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741891_1069 (size=4592) 2024-12-09T14:47:18,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741891_1069 (size=4592) 2024-12-09T14:47:18,316 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/.tmp/cf2/0e3956135d63446b8b6f35759fd78b5c as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/0e3956135d63446b8b6f35759fd78b5c 2024-12-09T14:47:18,324 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 10975ad07e6cf31ab36e8c23dc0846f3/cf2 of 10975ad07e6cf31ab36e8c23dc0846f3 into 0e3956135d63446b8b6f35759fd78b5c(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T14:47:18,324 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 10975ad07e6cf31ab36e8c23dc0846f3: 2024-12-09T14:47:18,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=10975ad07e6cf31ab36e8c23dc0846f3, source=2491923345a5,43689,1733755621826, destination=2491923345a5,37129,1733755621729, warming up region on 2491923345a5,37129,1733755621729 2024-12-09T14:47:18,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=10975ad07e6cf31ab36e8c23dc0846f3, source=2491923345a5,43689,1733755621826, destination=2491923345a5,37129,1733755621729, running balancer 2024-12-09T14:47:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE 2024-12-09T14:47:18,331 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE 2024-12-09T14:47:18,333 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=CLOSING, regionLocation=2491923345a5,43689,1733755621826 2024-12-09T14:47:18,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(7855): Warmup {ENCODED => 10975ad07e6cf31ab36e8c23dc0846f3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:18,334 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,335 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf1 2024-12-09T14:47:18,335 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:18,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE because future has completed 2024-12-09T14:47:18,336 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T14:47:18,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,43689,1733755621826}] 2024-12-09T14:47:18,343 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/21d1c01f806e42349817e88535f61c50 2024-12-09T14:47:18,349 INFO [StoreFileOpener-10975ad07e6cf31ab36e8c23dc0846f3-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 457ed6d84a2c4548b2f735fa7cf7c875 2024-12-09T14:47:18,349 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/457ed6d84a2c4548b2f735fa7cf7c875 2024-12-09T14:47:18,355 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/eef99cada17240548893e51d5c018ca2 2024-12-09T14:47:18,356 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:18,356 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,357 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf2 2024-12-09T14:47:18,357 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:18,366 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/0e3956135d63446b8b6f35759fd78b5c 2024-12-09T14:47:18,372 INFO [StoreFileOpener-10975ad07e6cf31ab36e8c23dc0846f3-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 520ecfa3ee8f4e04a4566db4e286a106 2024-12-09T14:47:18,372 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/520ecfa3ee8f4e04a4566db4e286a106 2024-12-09T14:47:18,373 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(1722): Closing 10975ad07e6cf31ab36e8c23dc0846f3, disabling compactions & flushes 2024-12-09T14:47:18,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. after waiting 0 ms 2024-12-09T14:47:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37129 {}] regionserver.HRegion(1676): Region close journal for 10975ad07e6cf31ab36e8c23dc0846f3: Waiting for close lock at 1733755638373Disabling compacts and flushes for region at 1733755638373Disabling writes for close at 1733755638373Writing region close event to WAL at 1733755638374 (+1 ms)Closed at 1733755638374 2024-12-09T14:47:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-09T14:47:18,491 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,491 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T14:47:18,491 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 10975ad07e6cf31ab36e8c23dc0846f3, disabling compactions & flushes 2024-12-09T14:47:18,491 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,491 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,491 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. after waiting 0 ms 2024-12-09T14:47:18,492 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,492 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/eef99cada17240548893e51d5c018ca2, hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/457ed6d84a2c4548b2f735fa7cf7c875] to archive 2024-12-09T14:47:18,495 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T14:47:18,500 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/eef99cada17240548893e51d5c018ca2 to hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/eef99cada17240548893e51d5c018ca2 2024-12-09T14:47:18,502 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/457ed6d84a2c4548b2f735fa7cf7c875 to hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/457ed6d84a2c4548b2f735fa7cf7c875 2024-12-09T14:47:18,513 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/520ecfa3ee8f4e04a4566db4e286a106] to archive 2024-12-09T14:47:18,514 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T14:47:18,516 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/520ecfa3ee8f4e04a4566db4e286a106 to hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/520ecfa3ee8f4e04a4566db4e286a106 2024-12-09T14:47:18,521 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-09T14:47:18,521 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,522 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 10975ad07e6cf31ab36e8c23dc0846f3: Waiting for close lock at 1733755638491Running coprocessor pre-close hooks at 1733755638491Disabling compacts and flushes for region at 1733755638491Disabling writes for close at 1733755638491Writing region close event to WAL at 1733755638517 (+26 ms)Running coprocessor post-close hooks at 1733755638521 (+4 ms)Closed at 1733755638521 2024-12-09T14:47:18,522 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 10975ad07e6cf31ab36e8c23dc0846f3 move to 2491923345a5,37129,1733755621729 record at close sequenceid=12 2024-12-09T14:47:18,524 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,524 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=CLOSED 2024-12-09T14:47:18,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,43689,1733755621826 because future has completed 2024-12-09T14:47:18,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T14:47:18,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,43689,1733755621826 in 192 msec 2024-12-09T14:47:18,531 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE; state=CLOSED, location=2491923345a5,37129,1733755621729; forceNewPlan=false, retain=false 2024-12-09T14:47:18,682 INFO [2491923345a5:40599 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T14:47:18,683 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=OPENING, regionLocation=2491923345a5,37129,1733755621729 2024-12-09T14:47:18,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE because future has completed 2024-12-09T14:47:18,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,37129,1733755621729}] 2024-12-09T14:47:18,850 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,850 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 10975ad07e6cf31ab36e8c23dc0846f3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:18,850 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,850 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:18,851 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,851 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,853 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,854 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf1 2024-12-09T14:47:18,855 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:18,863 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/21d1c01f806e42349817e88535f61c50 2024-12-09T14:47:18,863 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:18,863 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,865 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf2 2024-12-09T14:47:18,865 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:18,873 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/0e3956135d63446b8b6f35759fd78b5c 2024-12-09T14:47:18,873 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:18,873 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,874 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,876 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,877 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,877 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,878 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T14:47:18,880 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,880 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 10975ad07e6cf31ab36e8c23dc0846f3; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75007314, jitterRate=0.11769607663154602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T14:47:18,881 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,881 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 10975ad07e6cf31ab36e8c23dc0846f3: Running coprocessor pre-open hook at 1733755638851Writing region info on filesystem at 1733755638851Initializing all the Stores at 1733755638852 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755638852Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755638853 (+1 ms)Cleaning up temporary data from old regions at 1733755638877 (+24 ms)Running coprocessor post-open hooks at 1733755638881 (+4 ms)Region opened successfully at 1733755638881 2024-12-09T14:47:18,882 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., pid=12, masterSystemTime=1733755638842 2024-12-09T14:47:18,885 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,885 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,886 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=OPEN, openSeqNum=18, regionLocation=2491923345a5,37129,1733755621729 2024-12-09T14:47:18,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,37129,1733755621729 because future has completed 2024-12-09T14:47:18,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-09T14:47:18,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,37129,1733755621729 in 201 msec 2024-12-09T14:47:18,894 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, REOPEN/MOVE in 563 msec 2024-12-09T14:47:18,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:47:18,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:47:18,938 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 2491923345a5,37129,1733755621729: testing ***** 2024-12-09T14:47:18,938 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-09T14:47:18,941 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-09T14:47:18,942 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-09T14:47:18,944 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-09T14:47:18,945 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-09T14:47:18,952 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 394539352 }, "NonHeapMemoryUsage": { "committed": 171442176, "init": 7667712, "max": -1, "used": 168831328 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2491923345a5", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2074, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2491923345a5", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58381", "tag.serverName": "2491923345a5,43689,1733755621826", "tag.clusterId": "20c7aab3-df44-4d3a-95f0-b2f8c0a2e453", "tag.Context": "regionserver", "tag.Hostname": "2491923345a5", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733755621826, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 7, "Put_mean": 4, "Put_25th_percentile": 3, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-09T14:47:18,955 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40599 {}] master.MasterRpcServices(700): 2491923345a5,37129,1733755621729 reported a fatal error: ***** ABORTING region server 2491923345a5,37129,1733755621729: testing ***** 2024-12-09T14:47:18,957 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2491923345a5,37129,1733755621729' ***** 2024-12-09T14:47:18,957 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-09T14:47:18,958 INFO [RS:0;2491923345a5:37129 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:47:18,958 INFO [RS:0;2491923345a5:37129 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-09T14:47:18,958 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:47:18,958 INFO [RS:0;2491923345a5:37129 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-09T14:47:18,958 INFO [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(3091): Received CLOSE for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43689 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:49456 deadline: 1733755698958, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2491923345a5 port=37129 startCode=1733755621729. As of locationSeqNum=12. 2024-12-09T14:47:18,958 INFO [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(956): aborting server 2491923345a5,37129,1733755621729 2024-12-09T14:47:18,959 INFO [RS:0;2491923345a5:37129 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:47:18,959 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 10975ad07e6cf31ab36e8c23dc0846f3, disabling compactions & flushes 2024-12-09T14:47:18,959 INFO [RS:0;2491923345a5:37129 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2491923345a5:37129. 2024-12-09T14:47:18,959 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,959 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,43689,1733755621826, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,43689,1733755621826, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2491923345a5 port=37129 startCode=1733755621729. As of locationSeqNum=12. 2024-12-09T14:47:18,959 DEBUG [RS:0;2491923345a5:37129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:47:18,959 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,959 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,43689,1733755621826, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2491923345a5 port=37129 startCode=1733755621729. As of locationSeqNum=12. 2024-12-09T14:47:18,959 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,43689,1733755621826, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=2491923345a5 port=37129 startCode=1733755621729. As of locationSeqNum=12. 2024-12-09T14:47:18,959 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. after waiting 0 ms 2024-12-09T14:47:18,959 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,959 DEBUG [RS:0;2491923345a5:37129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:47:18,960 INFO [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T14:47:18,960 DEBUG [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(1325): Online Regions={10975ad07e6cf31ab36e8c23dc0846f3=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.} 2024-12-09T14:47:18,960 DEBUG [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(1351): Waiting on 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:18,961 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:18,961 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 10975ad07e6cf31ab36e8c23dc0846f3: Waiting for close lock at 1733755638959Running coprocessor pre-close hooks at 1733755638959Disabling compacts and flushes for region at 1733755638959Disabling writes for close at 1733755638959Writing region close event to WAL at 1733755638961 (+2 ms)Running coprocessor post-close hooks at 1733755638961Closed at 1733755638961 2024-12-09T14:47:18,962 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:19,013 INFO [regionserver/2491923345a5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:47:19,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2491923345a5,37129,1733755621729 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:19,072 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2491923345a5,37129,1733755621729 aborting 2024-12-09T14:47:19,072 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 2491923345a5,37129,1733755621729 aborting 2024-12-09T14:47:19,072 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=12 from cache 2024-12-09T14:47:19,161 INFO [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(976): stopping server 2491923345a5,37129,1733755621729; all regions closed. 2024-12-09T14:47:19,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741835_1011 (size=2288) 2024-12-09T14:47:19,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741835_1011 (size=2288) 2024-12-09T14:47:19,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741835_1011 (size=2288) 2024-12-09T14:47:19,167 DEBUG [RS:0;2491923345a5:37129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:47:19,167 INFO [RS:0;2491923345a5:37129 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:47:19,167 INFO [RS:0;2491923345a5:37129 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:47:19,167 INFO [RS:0;2491923345a5:37129 {}] hbase.ChoreService(370): Chore service for: regionserver/2491923345a5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T14:47:19,168 INFO [regionserver/2491923345a5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:47:19,168 INFO [RS:0;2491923345a5:37129 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:47:19,168 INFO [RS:0;2491923345a5:37129 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:47:19,168 INFO [RS:0;2491923345a5:37129 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:47:19,168 INFO [RS:0;2491923345a5:37129 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:47:19,169 INFO [RS:0;2491923345a5:37129 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37129 2024-12-09T14:47:19,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:47:19,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2491923345a5,37129,1733755621729 2024-12-09T14:47:19,220 INFO [RS:0;2491923345a5:37129 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:47:19,222 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2491923345a5,37129,1733755621729] 2024-12-09T14:47:19,236 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2491923345a5,37129,1733755621729 already deleted, retry=false 2024-12-09T14:47:19,237 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2491923345a5,37129,1733755621729 on 2491923345a5,40599,1733755621019 2024-12-09T14:47:19,247 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2491923345a5,37129,1733755621729, splitWal=true, meta=false 2024-12-09T14:47:19,250 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 2491923345a5,37129,1733755621729 (carryingMeta=false) 2491923345a5,37129,1733755621729/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7e46f0f7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T14:47:19,251 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2491923345a5,37129,1733755621729, splitWal=true, meta=false 2024-12-09T14:47:19,252 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 2491923345a5,37129,1733755621729 had 1 regions 2024-12-09T14:47:19,253 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2491923345a5,37129,1733755621729, splitWal=true, meta=false, isMeta: false 2024-12-09T14:47:19,254 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting 2024-12-09T14:47:19,255 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2491923345a5,37129,1733755621729 WAL count=1, meta=false 2024-12-09T14:47:19,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2491923345a5%2C37129%2C1733755621729.1733755623457}] 2024-12-09T14:47:19,262 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=2491923345a5,43689,1733755621826 2024-12-09T14:47:19,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2491923345a5%2C37129%2C1733755621729.1733755623457, worker=2491923345a5,43689,1733755621826}] 2024-12-09T14:47:19,281 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=18] 2024-12-09T14:47:19,283 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2491923345a5:37129 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2491923345a5/172.17.0.2:37129 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:19,284 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=18, error=java.net.ConnectException: Call to address=2491923345a5:37129 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2491923345a5/172.17.0.2:37129 2024-12-09T14:47:19,284 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-09T14:47:19,284 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=18 from cache 2024-12-09T14:47:19,284 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 2491923345a5:37129 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2491923345a5/172.17.0.2:37129 2024-12-09T14:47:19,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:47:19,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37129-0x1000b190a330001, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:47:19,329 INFO [RS:0;2491923345a5:37129 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:47:19,330 INFO [RS:0;2491923345a5:37129 {}] regionserver.HRegionServer(1031): Exiting; stopping=2491923345a5,37129,1733755621729; zookeeper connection closed. 2024-12-09T14:47:19,330 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@620557f8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@620557f8 2024-12-09T14:47:19,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43689 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-09T14:47:19,449 INFO [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457, size=2.2 K (2288bytes) 2024-12-09T14:47:19,449 INFO [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457 2024-12-09T14:47:19,450 INFO [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457 after 1ms 2024-12-09T14:47:19,475 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:19,475 INFO [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457 took 26ms 2024-12-09T14:47:19,481 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 10975ad07e6cf31ab36e8c23dc0846f3: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-09T14:47:19,482 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457 so closing down 2024-12-09T14:47:19,482 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:19,482 INFO [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:19,482 INFO [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-09T14:47:19,482 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457, journal: Splitting hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457, size=2.2 K (2288bytes) at 1733755639449Finishing writing output for hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457 so closing down at 1733755639482 (+33 ms)3 split writer threads finished at 1733755639482Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733755639482 2024-12-09T14:47:19,482 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457 2024-12-09T14:47:19,483 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2491923345a5:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-09T14:47:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40599 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-09T14:47:19,489 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting/2491923345a5%2C37129%2C1733755621729.1733755623457 to hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/oldWALs 2024-12-09T14:47:19,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T14:47:19,491 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2491923345a5%2C37129%2C1733755621729.1733755623457, worker=2491923345a5,43689,1733755621826 in 224 msec 2024-12-09T14:47:19,492 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=2491923345a5,43689,1733755621826 2024-12-09T14:47:19,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T14:47:19,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 2491923345a5%2C37129%2C1733755621729.1733755623457, worker=2491923345a5,43689,1733755621826 in 236 msec 2024-12-09T14:47:19,497 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting dir is empty, no logs to split. 2024-12-09T14:47:19,497 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2491923345a5,37129,1733755621729 WAL count=0, meta=false 2024-12-09T14:47:19,497 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2491923345a5,37129,1733755621729 WAL splitting is done? wals=0, meta=false 2024-12-09T14:47:19,499 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 2491923345a5,37129,1733755621729 failed, ignore...File hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/WALs/2491923345a5,37129,1733755621729-splitting does not exist. 2024-12-09T14:47:19,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, ASSIGN}] 2024-12-09T14:47:19,503 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, ASSIGN 2024-12-09T14:47:19,504 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T14:47:19,590 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=18] 2024-12-09T14:47:19,591 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 2491923345a5:37129 this server is in the failed servers list 2024-12-09T14:47:19,591 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=2491923345a5:37129 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2491923345a5:37129 2024-12-09T14:47:19,591 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 2491923345a5:37129 2024-12-09T14:47:19,591 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,37129,1733755621729, seqNum=18 from cache 2024-12-09T14:47:19,655 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(204): Hosts are {2491923345a5=0} racks are {/default-rack=0} 2024-12-09T14:47:19,655 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T14:47:19,656 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T14:47:19,656 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T14:47:19,656 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T14:47:19,656 INFO [2491923345a5:40599 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T14:47:19,656 INFO [2491923345a5:40599 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T14:47:19,656 DEBUG [2491923345a5:40599 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T14:47:19,657 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=OPENING, regionLocation=2491923345a5,44703,1733755621870 2024-12-09T14:47:19,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, ASSIGN because future has completed 2024-12-09T14:47:19,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,44703,1733755621870}] 2024-12-09T14:47:19,816 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T14:47:19,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44143, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T14:47:19,825 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:19,825 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 10975ad07e6cf31ab36e8c23dc0846f3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:19,826 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,826 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:19,827 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,827 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,829 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,831 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf1 2024-12-09T14:47:19,831 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:19,837 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf1/21d1c01f806e42349817e88535f61c50 2024-12-09T14:47:19,837 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:19,837 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,838 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10975ad07e6cf31ab36e8c23dc0846f3 columnFamilyName cf2 2024-12-09T14:47:19,838 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:19,844 DEBUG [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/cf2/0e3956135d63446b8b6f35759fd78b5c 2024-12-09T14:47:19,844 INFO [StoreOpener-10975ad07e6cf31ab36e8c23dc0846f3-1 {}] regionserver.HStore(327): Store=10975ad07e6cf31ab36e8c23dc0846f3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:19,844 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,845 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,846 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,847 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,847 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,848 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-09T14:47:19,849 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,850 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 10975ad07e6cf31ab36e8c23dc0846f3; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71415300, jitterRate=0.06417089700698853}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-09T14:47:19,850 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:19,851 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 10975ad07e6cf31ab36e8c23dc0846f3: Running coprocessor pre-open hook at 1733755639827Writing region info on filesystem at 1733755639827Initializing all the Stores at 1733755639828 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755639828Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755639829 (+1 ms)Cleaning up temporary data from old regions at 1733755639847 (+18 ms)Running coprocessor post-open hooks at 1733755639850 (+3 ms)Region opened successfully at 1733755639851 (+1 ms) 2024-12-09T14:47:19,852 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., pid=17, masterSystemTime=1733755639816 2024-12-09T14:47:19,854 DEBUG [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:19,854 INFO [RS_OPEN_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:19,855 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=10975ad07e6cf31ab36e8c23dc0846f3, regionState=OPEN, openSeqNum=18, regionLocation=2491923345a5,44703,1733755621870 2024-12-09T14:47:19,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,44703,1733755621870 because future has completed 2024-12-09T14:47:19,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T14:47:19,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 10975ad07e6cf31ab36e8c23dc0846f3, server=2491923345a5,44703,1733755621870 in 195 msec 2024-12-09T14:47:19,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-09T14:47:19,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=10975ad07e6cf31ab36e8c23dc0846f3, ASSIGN in 358 msec 2024-12-09T14:47:19,861 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 2491923345a5,37129,1733755621729 after splitting done 2024-12-09T14:47:19,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 2491923345a5,37129,1733755621729, splitWal=true, meta=false in 622 msec 2024-12-09T14:47:20,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3., hostname=2491923345a5,44703,1733755621870, seqNum=18] 2024-12-09T14:47:20,104 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T14:47:20,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T14:47:20,123 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=413 (was 413), OpenFileDescriptor=1051 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=216 (was 209) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7619 (was 7656) 2024-12-09T14:47:20,125 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-09T14:47:20,135 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=413, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=216, ProcessCount=11, AvailableMemoryMB=7618 2024-12-09T14:47:20,135 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-09T14:47:20,148 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:20,150 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:20,151 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:20,153 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57873448, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/hregion-57873448, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:20,164 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57873448/hregion-57873448.1733755640154, exclude list is [], retry=0 2024-12-09T14:47:20,167 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:20,167 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:20,167 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:20,190 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57873448/hregion-57873448.1733755640154 2024-12-09T14:47:20,190 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:20,191 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 14ac7ab47ec8e8621b5b6b84c1be3241, NAME => 'testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/hbase 2024-12-09T14:47:20,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741893_1071 (size=67) 2024-12-09T14:47:20,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741893_1071 (size=67) 2024-12-09T14:47:20,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741893_1071 (size=67) 2024-12-09T14:47:20,200 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:20,202 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,203 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ac7ab47ec8e8621b5b6b84c1be3241 columnFamilyName a 2024-12-09T14:47:20,203 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,204 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(327): Store=14ac7ab47ec8e8621b5b6b84c1be3241/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,204 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,205 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ac7ab47ec8e8621b5b6b84c1be3241 columnFamilyName b 2024-12-09T14:47:20,205 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,206 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(327): Store=14ac7ab47ec8e8621b5b6b84c1be3241/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,206 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,207 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ac7ab47ec8e8621b5b6b84c1be3241 columnFamilyName c 2024-12-09T14:47:20,207 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,207 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(327): Store=14ac7ab47ec8e8621b5b6b84c1be3241/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,208 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,209 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,209 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,210 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,210 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,211 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:20,212 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,213 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:20,214 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 14ac7ab47ec8e8621b5b6b84c1be3241; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65838089, jitterRate=-0.018936023116111755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:20,214 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 14ac7ab47ec8e8621b5b6b84c1be3241: Writing region info on filesystem at 1733755640200Initializing all the Stores at 1733755640201 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640201Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640201Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640201Cleaning up temporary data from old regions at 1733755640210 (+9 ms)Region opened successfully at 1733755640214 (+4 ms) 2024-12-09T14:47:20,215 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 14ac7ab47ec8e8621b5b6b84c1be3241, disabling compactions & flushes 2024-12-09T14:47:20,215 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. 2024-12-09T14:47:20,215 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. 2024-12-09T14:47:20,215 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. after waiting 0 ms 2024-12-09T14:47:20,215 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. 2024-12-09T14:47:20,215 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. 2024-12-09T14:47:20,215 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 14ac7ab47ec8e8621b5b6b84c1be3241: Waiting for close lock at 1733755640215Disabling compacts and flushes for region at 1733755640215Disabling writes for close at 1733755640215Writing region close event to WAL at 1733755640215Closed at 1733755640215 2024-12-09T14:47:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741892_1070 (size=175) 2024-12-09T14:47:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741892_1070 (size=175) 2024-12-09T14:47:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741892_1070 (size=175) 2024-12-09T14:47:20,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:20,219 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57873448:(num 1733755640154) 2024-12-09T14:47:20,219 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:20,221 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:20,231 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221, exclude list is [], retry=0 2024-12-09T14:47:20,234 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:20,234 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:20,235 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:20,257 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221 2024-12-09T14:47:20,257 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:20,257 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 14ac7ab47ec8e8621b5b6b84c1be3241, NAME => 'testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:20,257 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:20,257 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,257 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,259 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,260 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ac7ab47ec8e8621b5b6b84c1be3241 columnFamilyName a 2024-12-09T14:47:20,260 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,260 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(327): Store=14ac7ab47ec8e8621b5b6b84c1be3241/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,260 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,261 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ac7ab47ec8e8621b5b6b84c1be3241 columnFamilyName b 2024-12-09T14:47:20,261 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,261 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(327): Store=14ac7ab47ec8e8621b5b6b84c1be3241/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,261 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,262 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ac7ab47ec8e8621b5b6b84c1be3241 columnFamilyName c 2024-12-09T14:47:20,262 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,262 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(327): Store=14ac7ab47ec8e8621b5b6b84c1be3241/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,262 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,263 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,265 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,265 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,266 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:20,267 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,268 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 14ac7ab47ec8e8621b5b6b84c1be3241; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64445303, jitterRate=-0.03969015181064606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:20,268 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 14ac7ab47ec8e8621b5b6b84c1be3241: Writing region info on filesystem at 1733755640257Initializing all the Stores at 1733755640258 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640258Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640259 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640259Cleaning up temporary data from old regions at 1733755640265 (+6 ms)Region opened successfully at 1733755640268 (+3 ms) 2024-12-09T14:47:20,292 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 14ac7ab47ec8e8621b5b6b84c1be3241 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T14:47:20,306 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/a/1814e4ecdcf3402fa09641bb04b4faf3 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733755640268/Put/seqid=0 2024-12-09T14:47:20,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741895_1073 (size=5958) 2024-12-09T14:47:20,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741895_1073 (size=5958) 2024-12-09T14:47:20,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741895_1073 (size=5958) 2024-12-09T14:47:20,313 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/a/1814e4ecdcf3402fa09641bb04b4faf3 2024-12-09T14:47:20,330 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/b/0ea7155fbb0a410fa2486fb5840e62c4 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733755640276/Put/seqid=0 2024-12-09T14:47:20,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741896_1074 (size=5958) 2024-12-09T14:47:20,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741896_1074 (size=5958) 2024-12-09T14:47:20,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741896_1074 (size=5958) 2024-12-09T14:47:20,337 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/b/0ea7155fbb0a410fa2486fb5840e62c4 2024-12-09T14:47:20,361 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/c/f45d166604c84ab08aeef99600460a44 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733755640283/Put/seqid=0 2024-12-09T14:47:20,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741897_1075 (size=5958) 2024-12-09T14:47:20,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741897_1075 (size=5958) 2024-12-09T14:47:20,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741897_1075 (size=5958) 2024-12-09T14:47:20,368 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/c/f45d166604c84ab08aeef99600460a44 2024-12-09T14:47:20,376 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/a/1814e4ecdcf3402fa09641bb04b4faf3 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/a/1814e4ecdcf3402fa09641bb04b4faf3 2024-12-09T14:47:20,383 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/a/1814e4ecdcf3402fa09641bb04b4faf3, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T14:47:20,384 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/b/0ea7155fbb0a410fa2486fb5840e62c4 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/b/0ea7155fbb0a410fa2486fb5840e62c4 2024-12-09T14:47:20,391 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/b/0ea7155fbb0a410fa2486fb5840e62c4, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T14:47:20,392 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/c/f45d166604c84ab08aeef99600460a44 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/c/f45d166604c84ab08aeef99600460a44 2024-12-09T14:47:20,398 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/c/f45d166604c84ab08aeef99600460a44, entries=10, sequenceid=33, filesize=5.8 K 2024-12-09T14:47:20,400 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 14ac7ab47ec8e8621b5b6b84c1be3241 in 108ms, sequenceid=33, compaction requested=false 2024-12-09T14:47:20,400 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 14ac7ab47ec8e8621b5b6b84c1be3241: 2024-12-09T14:47:20,400 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 14ac7ab47ec8e8621b5b6b84c1be3241, disabling compactions & flushes 2024-12-09T14:47:20,400 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. 2024-12-09T14:47:20,400 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. 2024-12-09T14:47:20,400 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. after waiting 0 ms 2024-12-09T14:47:20,400 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. 2024-12-09T14:47:20,401 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. 2024-12-09T14:47:20,401 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 14ac7ab47ec8e8621b5b6b84c1be3241: Waiting for close lock at 1733755640400Disabling compacts and flushes for region at 1733755640400Disabling writes for close at 1733755640400Writing region close event to WAL at 1733755640401 (+1 ms)Closed at 1733755640401 2024-12-09T14:47:20,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741894_1072 (size=6311) 2024-12-09T14:47:20,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741894_1072 (size=6311) 2024-12-09T14:47:20,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741894_1072 (size=6311) 2024-12-09T14:47:20,408 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/b/0ea7155fbb0a410fa2486fb5840e62c4 to hdfs://localhost:35639/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/b/0ea7155fbb0a410fa2486fb5840e62c4 2024-12-09T14:47:20,423 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221, size=6.2 K (6311bytes) 2024-12-09T14:47:20,423 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221 2024-12-09T14:47:20,424 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221 after 1ms 2024-12-09T14:47:20,447 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:20,447 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221 took 24ms 2024-12-09T14:47:20,450 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221 so closing down 2024-12-09T14:47:20,450 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:20,451 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733755640221.temp 2024-12-09T14:47:20,473 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000003-wal.1733755640221.temp 2024-12-09T14:47:20,474 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:20,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741898_1076 (size=5575) 2024-12-09T14:47:20,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741898_1076 (size=5575) 2024-12-09T14:47:20,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741898_1076 (size=5575) 2024-12-09T14:47:20,483 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000003-wal.1733755640221.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:20,484 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000003-wal.1733755640221.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000032 2024-12-09T14:47:20,485 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-09T14:47:20,485 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221, journal: Splitting hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221, size=6.2 K (6311bytes) at 1733755640423Finishing writing output for hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221 so closing down at 1733755640450 (+27 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000003-wal.1733755640221.temp at 1733755640473 (+23 ms)3 split writer threads finished at 1733755640474 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000003-wal.1733755640221.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733755640483 (+9 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000003-wal.1733755640221.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000032 at 1733755640484 (+1 ms)Processed 32 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733755640485 (+1 ms) 2024-12-09T14:47:20,486 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640221 to hdfs://localhost:35639/hbase/oldWALs/wal.1733755640221 2024-12-09T14:47:20,487 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000032 2024-12-09T14:47:20,487 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:20,489 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:20,500 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640489, exclude list is [], retry=0 2024-12-09T14:47:20,502 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:20,502 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:20,503 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:20,523 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640489 2024-12-09T14:47:20,524 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429)] 2024-12-09T14:47:20,524 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 14ac7ab47ec8e8621b5b6b84c1be3241, NAME => 'testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:20,524 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:20,524 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,524 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,525 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,526 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ac7ab47ec8e8621b5b6b84c1be3241 columnFamilyName a 2024-12-09T14:47:20,526 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,530 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/a/1814e4ecdcf3402fa09641bb04b4faf3 2024-12-09T14:47:20,531 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(327): Store=14ac7ab47ec8e8621b5b6b84c1be3241/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,531 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,532 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ac7ab47ec8e8621b5b6b84c1be3241 columnFamilyName b 2024-12-09T14:47:20,532 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,533 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(327): Store=14ac7ab47ec8e8621b5b6b84c1be3241/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,533 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,534 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ac7ab47ec8e8621b5b6b84c1be3241 columnFamilyName c 2024-12-09T14:47:20,534 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,539 DEBUG [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/c/f45d166604c84ab08aeef99600460a44 2024-12-09T14:47:20,539 INFO [StoreOpener-14ac7ab47ec8e8621b5b6b84c1be3241-1 {}] regionserver.HStore(327): Store=14ac7ab47ec8e8621b5b6b84c1be3241/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,539 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,540 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,541 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,542 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000032 2024-12-09T14:47:20,567 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:20,568 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000032 2024-12-09T14:47:20,569 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 14ac7ab47ec8e8621b5b6b84c1be3241 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T14:47:20,581 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/b/11dea6ae79e94a9083c92ee830ba6c47 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733755640276/Put/seqid=0 2024-12-09T14:47:20,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741900_1078 (size=5958) 2024-12-09T14:47:20,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741900_1078 (size=5958) 2024-12-09T14:47:20,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741900_1078 (size=5958) 2024-12-09T14:47:20,588 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/b/11dea6ae79e94a9083c92ee830ba6c47 2024-12-09T14:47:20,593 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/.tmp/b/11dea6ae79e94a9083c92ee830ba6c47 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/b/11dea6ae79e94a9083c92ee830ba6c47 2024-12-09T14:47:20,598 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/b/11dea6ae79e94a9083c92ee830ba6c47, entries=10, sequenceid=32, filesize=5.8 K 2024-12-09T14:47:20,599 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 14ac7ab47ec8e8621b5b6b84c1be3241 in 30ms, sequenceid=32, compaction requested=false; wal=null 2024-12-09T14:47:20,600 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/0000000000000000032 2024-12-09T14:47:20,601 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,601 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,601 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:20,602 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 14ac7ab47ec8e8621b5b6b84c1be3241 2024-12-09T14:47:20,605 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/14ac7ab47ec8e8621b5b6b84c1be3241/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-09T14:47:20,606 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 14ac7ab47ec8e8621b5b6b84c1be3241; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66044068, jitterRate=-0.01586669683456421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:20,606 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 14ac7ab47ec8e8621b5b6b84c1be3241: Writing region info on filesystem at 1733755640524Initializing all the Stores at 1733755640525 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640525Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640525Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640525Obtaining lock to block concurrent updates at 1733755640569 (+44 ms)Preparing flush snapshotting stores in 14ac7ab47ec8e8621b5b6b84c1be3241 at 1733755640569Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733755640569Flushing stores of testReplayEditsWrittenViaHRegion,,1733755640149.14ac7ab47ec8e8621b5b6b84c1be3241. at 1733755640569Flushing 14ac7ab47ec8e8621b5b6b84c1be3241/b: creating writer at 1733755640569Flushing 14ac7ab47ec8e8621b5b6b84c1be3241/b: appending metadata at 1733755640580 (+11 ms)Flushing 14ac7ab47ec8e8621b5b6b84c1be3241/b: closing flushed file at 1733755640580Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@184a7fd5: reopening flushed file at 1733755640592 (+12 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 14ac7ab47ec8e8621b5b6b84c1be3241 in 30ms, sequenceid=32, compaction requested=false; wal=null at 1733755640599 (+7 ms)Cleaning up temporary data from old regions at 1733755640601 (+2 ms)Region opened successfully at 1733755640606 (+5 ms) 2024-12-09T14:47:20,625 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=422 (was 413) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:35650 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:51898 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:51754 [Waiting for operation #39] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:53676 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:35508 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:53772 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=216 (was 216), ProcessCount=11 (was 11), AvailableMemoryMB=7612 (was 7618) 2024-12-09T14:47:20,625 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-12-09T14:47:20,635 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=422, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=216, ProcessCount=11, AvailableMemoryMB=7612 2024-12-09T14:47:20,635 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-12-09T14:47:20,648 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:20,650 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:20,650 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:20,653 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73354993, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/hregion-73354993, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:20,664 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73354993/hregion-73354993.1733755640653, exclude list is [], retry=0 2024-12-09T14:47:20,667 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:20,667 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:20,667 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:20,693 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73354993/hregion-73354993.1733755640653 2024-12-09T14:47:20,693 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:20,693 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => be86e11b87c6a52aca10efa7835f76f8, NAME => 'testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/hbase 2024-12-09T14:47:20,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741902_1080 (size=68) 2024-12-09T14:47:20,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741902_1080 (size=68) 2024-12-09T14:47:20,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741902_1080 (size=68) 2024-12-09T14:47:20,703 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:20,704 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,705 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be86e11b87c6a52aca10efa7835f76f8 columnFamilyName a 2024-12-09T14:47:20,705 DEBUG [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,706 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(327): Store=be86e11b87c6a52aca10efa7835f76f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,706 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,708 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be86e11b87c6a52aca10efa7835f76f8 columnFamilyName b 2024-12-09T14:47:20,708 DEBUG [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,708 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(327): Store=be86e11b87c6a52aca10efa7835f76f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,708 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,710 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be86e11b87c6a52aca10efa7835f76f8 columnFamilyName c 2024-12-09T14:47:20,710 DEBUG [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,711 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(327): Store=be86e11b87c6a52aca10efa7835f76f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,711 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,712 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,712 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,713 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,713 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,714 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:20,715 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,717 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:20,717 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened be86e11b87c6a52aca10efa7835f76f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69241848, jitterRate=0.03178393840789795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:20,718 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for be86e11b87c6a52aca10efa7835f76f8: Writing region info on filesystem at 1733755640703Initializing all the Stores at 1733755640704 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640704Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640704Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640704Cleaning up temporary data from old regions at 1733755640713 (+9 ms)Region opened successfully at 1733755640718 (+5 ms) 2024-12-09T14:47:20,718 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing be86e11b87c6a52aca10efa7835f76f8, disabling compactions & flushes 2024-12-09T14:47:20,718 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. 2024-12-09T14:47:20,718 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. 2024-12-09T14:47:20,718 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. after waiting 0 ms 2024-12-09T14:47:20,718 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. 2024-12-09T14:47:20,718 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. 2024-12-09T14:47:20,718 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for be86e11b87c6a52aca10efa7835f76f8: Waiting for close lock at 1733755640718Disabling compacts and flushes for region at 1733755640718Disabling writes for close at 1733755640718Writing region close event to WAL at 1733755640718Closed at 1733755640718 2024-12-09T14:47:20,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741901_1079 (size=175) 2024-12-09T14:47:20,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741901_1079 (size=175) 2024-12-09T14:47:20,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741901_1079 (size=175) 2024-12-09T14:47:20,723 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:20,723 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73354993:(num 1733755640653) 2024-12-09T14:47:20,723 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:20,725 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:20,735 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725, exclude list is [], retry=0 2024-12-09T14:47:20,738 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:20,738 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:20,738 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:20,759 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725 2024-12-09T14:47:20,759 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:46429:46429)] 2024-12-09T14:47:20,821 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => be86e11b87c6a52aca10efa7835f76f8, NAME => 'testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:20,824 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,824 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:20,824 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,824 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,826 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,827 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be86e11b87c6a52aca10efa7835f76f8 columnFamilyName a 2024-12-09T14:47:20,827 DEBUG [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,827 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(327): Store=be86e11b87c6a52aca10efa7835f76f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,828 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,828 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be86e11b87c6a52aca10efa7835f76f8 columnFamilyName b 2024-12-09T14:47:20,828 DEBUG [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,829 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(327): Store=be86e11b87c6a52aca10efa7835f76f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,829 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,830 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be86e11b87c6a52aca10efa7835f76f8 columnFamilyName c 2024-12-09T14:47:20,830 DEBUG [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:20,830 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(327): Store=be86e11b87c6a52aca10efa7835f76f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:20,830 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,831 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,832 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,833 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,833 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,834 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:20,835 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,836 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened be86e11b87c6a52aca10efa7835f76f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65408928, jitterRate=-0.02533102035522461}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:20,836 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:20,836 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for be86e11b87c6a52aca10efa7835f76f8: Running coprocessor pre-open hook at 1733755640824Writing region info on filesystem at 1733755640824Initializing all the Stores at 1733755640825 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640825Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640825Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755640825Cleaning up temporary data from old regions at 1733755640833 (+8 ms)Running coprocessor post-open hooks at 1733755640836 (+3 ms)Region opened successfully at 1733755640836 2024-12-09T14:47:20,849 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing be86e11b87c6a52aca10efa7835f76f8 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-09T14:47:20,850 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for be86e11b87c6a52aca10efa7835f76f8/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:21,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-09T14:47:21,335 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-09T14:47:21,337 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-09T14:47:21,337 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-09T14:47:21,339 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-09T14:47:21,339 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-09T14:47:21,340 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-09T14:47:21,340 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-09T14:47:21,342 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-09T14:47:21,342 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-09T14:47:21,851 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for be86e11b87c6a52aca10efa7835f76f8/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:22,644 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T14:47:22,852 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for be86e11b87c6a52aca10efa7835f76f8/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:23,852 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for be86e11b87c6a52aca10efa7835f76f8/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:24,854 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for be86e11b87c6a52aca10efa7835f76f8/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:25,855 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for be86e11b87c6a52aca10efa7835f76f8/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:26,856 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for be86e11b87c6a52aca10efa7835f76f8/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:27,857 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for be86e11b87c6a52aca10efa7835f76f8/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:28,858 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for be86e11b87c6a52aca10efa7835f76f8/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:29,859 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for be86e11b87c6a52aca10efa7835f76f8/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:29,862 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for be86e11b87c6a52aca10efa7835f76f8: 2024-12-09T14:47:29,862 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. 2024-12-09T14:47:29,882 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for be86e11b87c6a52aca10efa7835f76f8: 2024-12-09T14:47:29,882 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-09T14:47:29,882 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing be86e11b87c6a52aca10efa7835f76f8, disabling compactions & flushes 2024-12-09T14:47:29,882 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. 2024-12-09T14:47:29,882 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. 2024-12-09T14:47:29,882 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. after waiting 0 ms 2024-12-09T14:47:29,882 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. 2024-12-09T14:47:29,883 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. 2024-12-09T14:47:29,883 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. 2024-12-09T14:47:29,883 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for be86e11b87c6a52aca10efa7835f76f8: Waiting for close lock at 1733755649882Running coprocessor pre-close hooks at 1733755649882Disabling compacts and flushes for region at 1733755649882Disabling writes for close at 1733755649882Writing region close event to WAL at 1733755649883 (+1 ms)Running coprocessor post-close hooks at 1733755649883Closed at 1733755649883 2024-12-09T14:47:29,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741903_1081 (size=4236) 2024-12-09T14:47:29,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741903_1081 (size=4236) 2024-12-09T14:47:29,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741903_1081 (size=4236) 2024-12-09T14:47:29,921 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725, size=4.1 K (4236bytes) 2024-12-09T14:47:29,921 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725 2024-12-09T14:47:29,921 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725 after 0ms 2024-12-09T14:47:29,944 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:29,944 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725 took 23ms 2024-12-09T14:47:29,946 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725 so closing down 2024-12-09T14:47:29,946 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:29,947 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733755640725.temp 2024-12-09T14:47:29,970 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000004-wal.1733755640725.temp 2024-12-09T14:47:29,970 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:29,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741904_1082 (size=3245) 2024-12-09T14:47:29,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741904_1082 (size=3245) 2024-12-09T14:47:29,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741904_1082 (size=3245) 2024-12-09T14:47:29,982 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000004-wal.1733755640725.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:29,983 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000004-wal.1733755640725.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000026 2024-12-09T14:47:29,984 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 40 ms; skipped=3; WAL=hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-09T14:47:29,984 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725, journal: Splitting hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725, size=4.1 K (4236bytes) at 1733755649921Finishing writing output for hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725 so closing down at 1733755649946 (+25 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000004-wal.1733755640725.temp at 1733755649970 (+24 ms)3 split writer threads finished at 1733755649970Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000004-wal.1733755640725.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733755649982 (+12 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000004-wal.1733755640725.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000026 at 1733755649983 (+1 ms)Processed 23 edits across 1 Regions in 40 ms; skipped=3; WAL=hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733755649984 (+1 ms) 2024-12-09T14:47:29,985 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755640725 to hdfs://localhost:35639/hbase/oldWALs/wal.1733755640725 2024-12-09T14:47:29,986 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000026 2024-12-09T14:47:29,986 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:29,987 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:29,999 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755649988, exclude list is [], retry=0 2024-12-09T14:47:30,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:30,002 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:30,002 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:30,022 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755649988 2024-12-09T14:47:30,023 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:30,023 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => be86e11b87c6a52aca10efa7835f76f8, NAME => 'testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:30,023 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,023 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:30,023 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,024 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,025 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,026 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be86e11b87c6a52aca10efa7835f76f8 columnFamilyName a 2024-12-09T14:47:30,026 DEBUG [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:30,026 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(327): Store=be86e11b87c6a52aca10efa7835f76f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:30,027 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,027 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be86e11b87c6a52aca10efa7835f76f8 columnFamilyName b 2024-12-09T14:47:30,027 DEBUG [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:30,028 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(327): Store=be86e11b87c6a52aca10efa7835f76f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:30,028 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,028 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be86e11b87c6a52aca10efa7835f76f8 columnFamilyName c 2024-12-09T14:47:30,028 DEBUG [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:30,029 INFO [StoreOpener-be86e11b87c6a52aca10efa7835f76f8-1 {}] regionserver.HStore(327): Store=be86e11b87c6a52aca10efa7835f76f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:30,029 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,029 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,030 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,031 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000026 2024-12-09T14:47:30,053 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:30,055 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000026 2024-12-09T14:47:30,055 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing be86e11b87c6a52aca10efa7835f76f8 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-09T14:47:30,068 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/.tmp/a/579d9b0b10384997a68be591327d678d is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733755649871/Put/seqid=0 2024-12-09T14:47:30,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741906_1084 (size=5523) 2024-12-09T14:47:30,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741906_1084 (size=5523) 2024-12-09T14:47:30,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741906_1084 (size=5523) 2024-12-09T14:47:30,075 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/.tmp/a/579d9b0b10384997a68be591327d678d 2024-12-09T14:47:30,093 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/.tmp/b/f9751596bb8c4297b932afe49e745935 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733755649863/Put/seqid=0 2024-12-09T14:47:30,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741907_1085 (size=5524) 2024-12-09T14:47:30,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741907_1085 (size=5524) 2024-12-09T14:47:30,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741907_1085 (size=5524) 2024-12-09T14:47:30,099 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/.tmp/b/f9751596bb8c4297b932afe49e745935 2024-12-09T14:47:30,116 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/.tmp/c/81c7ff3a20b94d908097baf00bef5f08 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733755649867/Put/seqid=0 2024-12-09T14:47:30,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741908_1086 (size=5457) 2024-12-09T14:47:30,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741908_1086 (size=5457) 2024-12-09T14:47:30,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741908_1086 (size=5457) 2024-12-09T14:47:30,123 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/.tmp/c/81c7ff3a20b94d908097baf00bef5f08 2024-12-09T14:47:30,130 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/.tmp/a/579d9b0b10384997a68be591327d678d as hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/a/579d9b0b10384997a68be591327d678d 2024-12-09T14:47:30,135 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/a/579d9b0b10384997a68be591327d678d, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T14:47:30,136 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/.tmp/b/f9751596bb8c4297b932afe49e745935 as hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/b/f9751596bb8c4297b932afe49e745935 2024-12-09T14:47:30,141 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/b/f9751596bb8c4297b932afe49e745935, entries=7, sequenceid=26, filesize=5.4 K 2024-12-09T14:47:30,142 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/.tmp/c/81c7ff3a20b94d908097baf00bef5f08 as hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/c/81c7ff3a20b94d908097baf00bef5f08 2024-12-09T14:47:30,148 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/c/81c7ff3a20b94d908097baf00bef5f08, entries=6, sequenceid=26, filesize=5.3 K 2024-12-09T14:47:30,149 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for be86e11b87c6a52aca10efa7835f76f8 in 94ms, sequenceid=26, compaction requested=false; wal=null 2024-12-09T14:47:30,149 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/0000000000000000026 2024-12-09T14:47:30,150 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,150 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,151 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:30,153 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,155 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsAfterAbortingFlush/be86e11b87c6a52aca10efa7835f76f8/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-09T14:47:30,156 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened be86e11b87c6a52aca10efa7835f76f8; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72976308, jitterRate=0.08743172883987427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:30,156 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be86e11b87c6a52aca10efa7835f76f8 2024-12-09T14:47:30,157 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for be86e11b87c6a52aca10efa7835f76f8: Running coprocessor pre-open hook at 1733755650024Writing region info on filesystem at 1733755650024Initializing all the Stores at 1733755650025 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755650025Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755650025Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755650025Obtaining lock to block concurrent updates at 1733755650055 (+30 ms)Preparing flush snapshotting stores in be86e11b87c6a52aca10efa7835f76f8 at 1733755650055Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733755650055Flushing stores of testReplayEditsAfterAbortingFlush,,1733755640649.be86e11b87c6a52aca10efa7835f76f8. at 1733755650055Flushing be86e11b87c6a52aca10efa7835f76f8/a: creating writer at 1733755650055Flushing be86e11b87c6a52aca10efa7835f76f8/a: appending metadata at 1733755650068 (+13 ms)Flushing be86e11b87c6a52aca10efa7835f76f8/a: closing flushed file at 1733755650068Flushing be86e11b87c6a52aca10efa7835f76f8/b: creating writer at 1733755650081 (+13 ms)Flushing be86e11b87c6a52aca10efa7835f76f8/b: appending metadata at 1733755650093 (+12 ms)Flushing be86e11b87c6a52aca10efa7835f76f8/b: closing flushed file at 1733755650093Flushing be86e11b87c6a52aca10efa7835f76f8/c: creating writer at 1733755650104 (+11 ms)Flushing be86e11b87c6a52aca10efa7835f76f8/c: appending metadata at 1733755650116 (+12 ms)Flushing be86e11b87c6a52aca10efa7835f76f8/c: closing flushed file at 1733755650116Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@665f0891: reopening flushed file at 1733755650129 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3647b16a: reopening flushed file at 1733755650135 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17db5ba9: reopening flushed file at 1733755650142 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for be86e11b87c6a52aca10efa7835f76f8 in 94ms, sequenceid=26, compaction requested=false; wal=null at 1733755650149 (+7 ms)Cleaning up temporary data from old regions at 1733755650151 (+2 ms)Running coprocessor post-open hooks at 1733755650156 (+5 ms)Region opened successfully at 1733755650157 (+1 ms) 2024-12-09T14:47:30,177 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420 (was 422), OpenFileDescriptor=1183 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 216), ProcessCount=11 (was 11), AvailableMemoryMB=7605 (was 7612) 2024-12-09T14:47:30,177 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-09T14:47:30,189 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=420, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=7606 2024-12-09T14:47:30,189 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-09T14:47:30,199 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T14:47:30,202 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:30,204 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:30,204 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:30,207 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-97164545, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/hregion-97164545, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:30,217 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-97164545/hregion-97164545.1733755650207, exclude list is [], retry=0 2024-12-09T14:47:30,220 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:30,220 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:30,220 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:30,241 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-97164545/hregion-97164545.1733755650207 2024-12-09T14:47:30,241 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:30,241 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e0909085b7cb0399e4d0a13844796855, NAME => 'testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/hbase 2024-12-09T14:47:30,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741910_1088 (size=61) 2024-12-09T14:47:30,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741910_1088 (size=61) 2024-12-09T14:47:30,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741910_1088 (size=61) 2024-12-09T14:47:30,249 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:30,250 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,251 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0909085b7cb0399e4d0a13844796855 columnFamilyName a 2024-12-09T14:47:30,251 DEBUG [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:30,252 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] regionserver.HStore(327): Store=e0909085b7cb0399e4d0a13844796855/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:30,252 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,253 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,253 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,253 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,253 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,255 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,257 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:30,257 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e0909085b7cb0399e4d0a13844796855; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67325992, jitterRate=0.003235459327697754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:47:30,258 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e0909085b7cb0399e4d0a13844796855: Writing region info on filesystem at 1733755650249Initializing all the Stores at 1733755650250 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755650250Cleaning up temporary data from old regions at 1733755650253 (+3 ms)Region opened successfully at 1733755650258 (+5 ms) 2024-12-09T14:47:30,258 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e0909085b7cb0399e4d0a13844796855, disabling compactions & flushes 2024-12-09T14:47:30,258 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. 2024-12-09T14:47:30,258 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. 2024-12-09T14:47:30,258 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. after waiting 0 ms 2024-12-09T14:47:30,258 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. 2024-12-09T14:47:30,259 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. 2024-12-09T14:47:30,259 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e0909085b7cb0399e4d0a13844796855: Waiting for close lock at 1733755650258Disabling compacts and flushes for region at 1733755650258Disabling writes for close at 1733755650258Writing region close event to WAL at 1733755650258Closed at 1733755650258 2024-12-09T14:47:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741909_1087 (size=175) 2024-12-09T14:47:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741909_1087 (size=175) 2024-12-09T14:47:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741909_1087 (size=175) 2024-12-09T14:47:30,263 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:30,263 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-97164545:(num 1733755650207) 2024-12-09T14:47:30,264 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:30,265 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:30,276 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265, exclude list is [], retry=0 2024-12-09T14:47:30,278 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:30,279 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:30,279 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:30,300 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265 2024-12-09T14:47:30,301 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:46429:46429)] 2024-12-09T14:47:30,301 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e0909085b7cb0399e4d0a13844796855, NAME => 'testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:30,301 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:30,301 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,301 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,302 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,303 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0909085b7cb0399e4d0a13844796855 columnFamilyName a 2024-12-09T14:47:30,303 DEBUG [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:30,304 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] regionserver.HStore(327): Store=e0909085b7cb0399e4d0a13844796855/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:30,304 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,304 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,306 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,306 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,307 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,308 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e0909085b7cb0399e4d0a13844796855; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62912415, jitterRate=-0.0625319629907608}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:47:30,308 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e0909085b7cb0399e4d0a13844796855: Writing region info on filesystem at 1733755650301Initializing all the Stores at 1733755650302 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755650302Cleaning up temporary data from old regions at 1733755650306 (+4 ms)Region opened successfully at 1733755650308 (+2 ms) 2024-12-09T14:47:30,315 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e0909085b7cb0399e4d0a13844796855, disabling compactions & flushes 2024-12-09T14:47:30,315 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. 2024-12-09T14:47:30,315 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. 2024-12-09T14:47:30,315 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. after waiting 0 ms 2024-12-09T14:47:30,315 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. 2024-12-09T14:47:30,316 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. 2024-12-09T14:47:30,316 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. 2024-12-09T14:47:30,316 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e0909085b7cb0399e4d0a13844796855: Waiting for close lock at 1733755650315Disabling compacts and flushes for region at 1733755650315Disabling writes for close at 1733755650315Writing region close event to WAL at 1733755650316 (+1 ms)Closed at 1733755650316 2024-12-09T14:47:30,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741911_1089 (size=1795) 2024-12-09T14:47:30,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741911_1089 (size=1795) 2024-12-09T14:47:30,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741911_1089 (size=1795) 2024-12-09T14:47:30,333 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265, size=1.8 K (1795bytes) 2024-12-09T14:47:30,333 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265 2024-12-09T14:47:30,334 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265 after 1ms 2024-12-09T14:47:30,366 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:30,366 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265 took 33ms 2024-12-09T14:47:30,368 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265 so closing down 2024-12-09T14:47:30,368 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:30,369 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733755650265.temp 2024-12-09T14:47:30,390 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000003-wal.1733755650265.temp 2024-12-09T14:47:30,391 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:30,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741912_1090 (size=1795) 2024-12-09T14:47:30,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741912_1090 (size=1795) 2024-12-09T14:47:30,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741912_1090 (size=1795) 2024-12-09T14:47:30,399 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000003-wal.1733755650265.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:30,400 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000003-wal.1733755650265.temp to hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000012 2024-12-09T14:47:30,400 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-09T14:47:30,401 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265, journal: Splitting hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265, size=1.8 K (1795bytes) at 1733755650333Finishing writing output for hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265 so closing down at 1733755650368 (+35 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000003-wal.1733755650265.temp at 1733755650390 (+22 ms)3 split writer threads finished at 1733755650391 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000003-wal.1733755650265.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733755650399 (+8 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000003-wal.1733755650265.temp to hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000012 at 1733755650400 (+1 ms)Processed 10 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733755650400 2024-12-09T14:47:30,402 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650265 to hdfs://localhost:35639/hbase/oldWALs/wal.1733755650265 2024-12-09T14:47:30,403 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000012 2024-12-09T14:47:30,429 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:30,713 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:30,715 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:30,725 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650715, exclude list is [], retry=0 2024-12-09T14:47:30,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:30,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:30,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:30,748 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650715 2024-12-09T14:47:30,748 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:30,749 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e0909085b7cb0399e4d0a13844796855, NAME => 'testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:30,749 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:30,749 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,749 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,751 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,753 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0909085b7cb0399e4d0a13844796855 columnFamilyName a 2024-12-09T14:47:30,753 DEBUG [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:30,754 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] regionserver.HStore(327): Store=e0909085b7cb0399e4d0a13844796855/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:30,754 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,755 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,756 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,757 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000012 2024-12-09T14:47:30,780 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:30,781 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000012 2024-12-09T14:47:30,781 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e0909085b7cb0399e4d0a13844796855 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-09T14:47:30,798 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/.tmp/a/fd51cbaf3a5b4e2aba6104e58075085b is 79, key is testDatalossWhenInputError/a:x0/1733755650308/Put/seqid=0 2024-12-09T14:47:30,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741914_1092 (size=5808) 2024-12-09T14:47:30,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741914_1092 (size=5808) 2024-12-09T14:47:30,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741914_1092 (size=5808) 2024-12-09T14:47:30,807 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/.tmp/a/fd51cbaf3a5b4e2aba6104e58075085b 2024-12-09T14:47:30,815 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/.tmp/a/fd51cbaf3a5b4e2aba6104e58075085b as hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/a/fd51cbaf3a5b4e2aba6104e58075085b 2024-12-09T14:47:30,823 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/a/fd51cbaf3a5b4e2aba6104e58075085b, entries=10, sequenceid=12, filesize=5.7 K 2024-12-09T14:47:30,823 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for e0909085b7cb0399e4d0a13844796855 in 42ms, sequenceid=12, compaction requested=false; wal=null 2024-12-09T14:47:30,824 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/0000000000000000012 2024-12-09T14:47:30,824 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,824 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,827 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,829 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-09T14:47:30,829 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e0909085b7cb0399e4d0a13844796855; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74889233, jitterRate=0.1159365326166153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:47:30,830 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e0909085b7cb0399e4d0a13844796855: Writing region info on filesystem at 1733755650749Initializing all the Stores at 1733755650751 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755650751Obtaining lock to block concurrent updates at 1733755650781 (+30 ms)Preparing flush snapshotting stores in e0909085b7cb0399e4d0a13844796855 at 1733755650781Finished memstore snapshotting testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733755650781Flushing stores of testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855. at 1733755650781Flushing e0909085b7cb0399e4d0a13844796855/a: creating writer at 1733755650781Flushing e0909085b7cb0399e4d0a13844796855/a: appending metadata at 1733755650797 (+16 ms)Flushing e0909085b7cb0399e4d0a13844796855/a: closing flushed file at 1733755650797Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d18c84c: reopening flushed file at 1733755650814 (+17 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for e0909085b7cb0399e4d0a13844796855 in 42ms, sequenceid=12, compaction requested=false; wal=null at 1733755650823 (+9 ms)Cleaning up temporary data from old regions at 1733755650824 (+1 ms)Region opened successfully at 1733755650830 (+6 ms) 2024-12-09T14:47:30,833 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e0909085b7cb0399e4d0a13844796855, NAME => 'testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:30,833 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733755650203.e0909085b7cb0399e4d0a13844796855.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:30,833 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,833 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,834 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,835 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e0909085b7cb0399e4d0a13844796855 columnFamilyName a 2024-12-09T14:47:30,835 DEBUG [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:30,841 DEBUG [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/a/fd51cbaf3a5b4e2aba6104e58075085b 2024-12-09T14:47:30,842 INFO [StoreOpener-e0909085b7cb0399e4d0a13844796855-1 {}] regionserver.HStore(327): Store=e0909085b7cb0399e4d0a13844796855/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:30,842 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,843 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,844 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,844 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,845 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,846 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e0909085b7cb0399e4d0a13844796855 2024-12-09T14:47:30,849 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testDatalossWhenInputError/e0909085b7cb0399e4d0a13844796855/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-09T14:47:30,850 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e0909085b7cb0399e4d0a13844796855; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72872421, jitterRate=0.08588369190692902}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T14:47:30,850 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e0909085b7cb0399e4d0a13844796855: Writing region info on filesystem at 1733755650833Initializing all the Stores at 1733755650834 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755650834Cleaning up temporary data from old regions at 1733755650845 (+11 ms)Region opened successfully at 1733755650850 (+5 ms) 2024-12-09T14:47:30,867 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=429 (was 420) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:59710 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:55048 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:59784 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:54506 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:55096 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:54450 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2135414708-172.17.0.2-1733755617276:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=7595 (was 7606) 2024-12-09T14:47:30,867 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-09T14:47:30,878 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=429, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=7595 2024-12-09T14:47:30,878 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-09T14:47:30,892 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:30,893 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:30,894 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:30,896 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-62926301, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/hregion-62926301, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:30,907 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-62926301/hregion-62926301.1733755650897, exclude list is [], retry=0 2024-12-09T14:47:30,909 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:30,909 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:30,910 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:30,932 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-62926301/hregion-62926301.1733755650897 2024-12-09T14:47:30,933 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:30,933 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b10cb8ed5eae23f4d66ef929dc5d1884, NAME => 'testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/hbase 2024-12-09T14:47:30,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741916_1094 (size=63) 2024-12-09T14:47:30,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741916_1094 (size=63) 2024-12-09T14:47:30,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741916_1094 (size=63) 2024-12-09T14:47:30,942 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:30,944 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:30,945 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b10cb8ed5eae23f4d66ef929dc5d1884 columnFamilyName a 2024-12-09T14:47:30,945 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:30,945 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(327): Store=b10cb8ed5eae23f4d66ef929dc5d1884/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:30,945 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:30,947 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b10cb8ed5eae23f4d66ef929dc5d1884 columnFamilyName b 2024-12-09T14:47:30,947 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:30,947 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(327): Store=b10cb8ed5eae23f4d66ef929dc5d1884/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:30,947 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:30,948 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b10cb8ed5eae23f4d66ef929dc5d1884 columnFamilyName c 2024-12-09T14:47:30,948 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:30,949 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(327): Store=b10cb8ed5eae23f4d66ef929dc5d1884/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:30,949 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:30,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:30,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:30,951 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:30,951 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:30,952 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:30,953 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:30,955 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:30,955 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b10cb8ed5eae23f4d66ef929dc5d1884; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72145431, jitterRate=0.07505069673061371}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:30,956 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b10cb8ed5eae23f4d66ef929dc5d1884: Writing region info on filesystem at 1733755650942Initializing all the Stores at 1733755650943 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755650943Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755650943Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755650943Cleaning up temporary data from old regions at 1733755650951 (+8 ms)Region opened successfully at 1733755650956 (+5 ms) 2024-12-09T14:47:30,956 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b10cb8ed5eae23f4d66ef929dc5d1884, disabling compactions & flushes 2024-12-09T14:47:30,956 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. 2024-12-09T14:47:30,956 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. 2024-12-09T14:47:30,956 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. after waiting 0 ms 2024-12-09T14:47:30,956 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. 2024-12-09T14:47:30,956 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. 2024-12-09T14:47:30,956 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b10cb8ed5eae23f4d66ef929dc5d1884: Waiting for close lock at 1733755650956Disabling compacts and flushes for region at 1733755650956Disabling writes for close at 1733755650956Writing region close event to WAL at 1733755650956Closed at 1733755650956 2024-12-09T14:47:30,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741915_1093 (size=175) 2024-12-09T14:47:30,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741915_1093 (size=175) 2024-12-09T14:47:30,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741915_1093 (size=175) 2024-12-09T14:47:30,966 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:30,966 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-62926301:(num 1733755650897) 2024-12-09T14:47:30,966 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:30,968 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:30,979 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968, exclude list is [], retry=0 2024-12-09T14:47:30,981 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:30,981 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:30,982 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:31,022 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 2024-12-09T14:47:31,022 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429)] 2024-12-09T14:47:31,023 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b10cb8ed5eae23f4d66ef929dc5d1884, NAME => 'testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:31,023 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:31,023 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,023 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,024 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,025 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b10cb8ed5eae23f4d66ef929dc5d1884 columnFamilyName a 2024-12-09T14:47:31,025 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:31,025 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(327): Store=b10cb8ed5eae23f4d66ef929dc5d1884/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:31,025 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,026 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b10cb8ed5eae23f4d66ef929dc5d1884 columnFamilyName b 2024-12-09T14:47:31,026 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:31,027 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(327): Store=b10cb8ed5eae23f4d66ef929dc5d1884/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:31,027 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,027 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b10cb8ed5eae23f4d66ef929dc5d1884 columnFamilyName c 2024-12-09T14:47:31,027 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:31,028 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(327): Store=b10cb8ed5eae23f4d66ef929dc5d1884/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:31,028 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,029 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,030 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,030 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,031 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,031 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:31,032 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:31,033 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b10cb8ed5eae23f4d66ef929dc5d1884; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67508138, jitterRate=0.005949646234512329}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:31,034 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b10cb8ed5eae23f4d66ef929dc5d1884: Writing region info on filesystem at 1733755651023Initializing all the Stores at 1733755651024 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755651024Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755651024Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755651024Cleaning up temporary data from old regions at 1733755651031 (+7 ms)Region opened successfully at 1733755651034 (+3 ms) 2024-12-09T14:47:31,039 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733755651038/Put/seqid=0 2024-12-09T14:47:31,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741918_1096 (size=4875) 2024-12-09T14:47:31,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741918_1096 (size=4875) 2024-12-09T14:47:31,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741918_1096 (size=4875) 2024-12-09T14:47:31,048 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733755651048/Put/seqid=0 2024-12-09T14:47:31,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741919_1097 (size=4875) 2024-12-09T14:47:31,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741919_1097 (size=4875) 2024-12-09T14:47:31,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741919_1097 (size=4875) 2024-12-09T14:47:31,059 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733755651059/Put/seqid=0 2024-12-09T14:47:31,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741920_1098 (size=4875) 2024-12-09T14:47:31,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741920_1098 (size=4875) 2024-12-09T14:47:31,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741920_1098 (size=4875) 2024-12-09T14:47:31,065 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in b10cb8ed5eae23f4d66ef929dc5d1884/a 2024-12-09T14:47:31,069 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-09T14:47:31,069 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T14:47:31,069 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in b10cb8ed5eae23f4d66ef929dc5d1884/a 2024-12-09T14:47:31,073 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-09T14:47:31,074 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T14:47:31,074 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in b10cb8ed5eae23f4d66ef929dc5d1884/a 2024-12-09T14:47:31,078 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-09T14:47:31,078 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-09T14:47:31,078 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b10cb8ed5eae23f4d66ef929dc5d1884 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-09T14:47:31,093 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/.tmp/a/9d07f04d3edb4c4bbd9b3e602f8a4f92 is 55, key is testCompactedBulkLoadedFiles/a:a/1733755651035/Put/seqid=0 2024-12-09T14:47:31,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741921_1099 (size=5107) 2024-12-09T14:47:31,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741921_1099 (size=5107) 2024-12-09T14:47:31,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741921_1099 (size=5107) 2024-12-09T14:47:31,100 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/.tmp/a/9d07f04d3edb4c4bbd9b3e602f8a4f92 2024-12-09T14:47:31,106 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/.tmp/a/9d07f04d3edb4c4bbd9b3e602f8a4f92 as hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/9d07f04d3edb4c4bbd9b3e602f8a4f92 2024-12-09T14:47:31,111 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/9d07f04d3edb4c4bbd9b3e602f8a4f92, entries=1, sequenceid=4, filesize=5.0 K 2024-12-09T14:47:31,112 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for b10cb8ed5eae23f4d66ef929dc5d1884 in 34ms, sequenceid=4, compaction requested=false 2024-12-09T14:47:31,112 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b10cb8ed5eae23f4d66ef929dc5d1884: 2024-12-09T14:47:31,114 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_ 2024-12-09T14:47:31,115 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_ 2024-12-09T14:47:31,116 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/928312febd804a5e875ddcdcd289eac3_SeqId_4_ 2024-12-09T14:47:31,116 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile0 into b10cb8ed5eae23f4d66ef929dc5d1884/a as hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_ - updating store file list. 2024-12-09T14:47:31,121 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:47:31,121 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_ into b10cb8ed5eae23f4d66ef929dc5d1884/a 2024-12-09T14:47:31,121 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile0 into b10cb8ed5eae23f4d66ef929dc5d1884/a (new location: hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_) 2024-12-09T14:47:31,122 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile1 into b10cb8ed5eae23f4d66ef929dc5d1884/a as hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_ - updating store file list. 2024-12-09T14:47:31,126 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:47:31,126 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_ into b10cb8ed5eae23f4d66ef929dc5d1884/a 2024-12-09T14:47:31,126 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile1 into b10cb8ed5eae23f4d66ef929dc5d1884/a (new location: hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_) 2024-12-09T14:47:31,127 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile2 into b10cb8ed5eae23f4d66ef929dc5d1884/a as hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/928312febd804a5e875ddcdcd289eac3_SeqId_4_ - updating store file list. 2024-12-09T14:47:31,131 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 928312febd804a5e875ddcdcd289eac3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:47:31,131 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/928312febd804a5e875ddcdcd289eac3_SeqId_4_ into b10cb8ed5eae23f4d66ef929dc5d1884/a 2024-12-09T14:47:31,131 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:35639/hbase/testCompactedBulkLoadedFiles/hfile2 into b10cb8ed5eae23f4d66ef929dc5d1884/a (new location: hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/928312febd804a5e875ddcdcd289eac3_SeqId_4_) 2024-12-09T14:47:31,138 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T14:47:31,138 DEBUG [Time-limited test {}] regionserver.HStore(1541): b10cb8ed5eae23f4d66ef929dc5d1884/a is initiating major compaction (all files) 2024-12-09T14:47:31,138 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of b10cb8ed5eae23f4d66ef929dc5d1884/a in testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. 2024-12-09T14:47:31,138 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/9d07f04d3edb4c4bbd9b3e602f8a4f92, hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_, hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_, hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/928312febd804a5e875ddcdcd289eac3_SeqId_4_] into tmpdir=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/.tmp, totalSize=19.3 K 2024-12-09T14:47:31,138 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9d07f04d3edb4c4bbd9b3e602f8a4f92, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733755651035 2024-12-09T14:47:31,139 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T14:47:31,139 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T14:47:31,140 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 928312febd804a5e875ddcdcd289eac3_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-09T14:47:31,151 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/.tmp/a/d2f455471873437c86489a23ab2affba is 55, key is testCompactedBulkLoadedFiles/a:a/1733755651035/Put/seqid=0 2024-12-09T14:47:31,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741922_1100 (size=6154) 2024-12-09T14:47:31,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741922_1100 (size=6154) 2024-12-09T14:47:31,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741922_1100 (size=6154) 2024-12-09T14:47:31,164 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/.tmp/a/d2f455471873437c86489a23ab2affba as hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/d2f455471873437c86489a23ab2affba 2024-12-09T14:47:31,171 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in b10cb8ed5eae23f4d66ef929dc5d1884/a of b10cb8ed5eae23f4d66ef929dc5d1884 into d2f455471873437c86489a23ab2affba(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T14:47:31,172 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for b10cb8ed5eae23f4d66ef929dc5d1884: 2024-12-09T14:47:31,172 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T14:47:31,172 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-09T14:47:31,201 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968, size=0 (0bytes) 2024-12-09T14:47:31,201 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 might be still open, length is 0 2024-12-09T14:47:31,201 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 2024-12-09T14:47:31,201 WARN [IPC Server handler 3 on default port 35639 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-09T14:47:31,201 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 after 0ms 2024-12-09T14:47:31,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-09T14:47:31,336 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-09T14:47:31,338 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-09T14:47:31,338 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-09T14:47:33,189 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:47820 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33317:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47820 dst: /127.0.0.1:33317 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33317 remote=/127.0.0.1:47820]. Total timeout mills is 60000, 57983 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:33,190 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:50164 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37163:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50164 dst: /127.0.0.1:37163 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:33,190 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:45778 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:46775:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45778 dst: /127.0.0.1:46775 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:33,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741917_1101 (size=1794) 2024-12-09T14:47:33,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741917_1101 (size=1794) 2024-12-09T14:47:35,203 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 after 4002ms 2024-12-09T14:47:35,277 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:35,277 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 took 4077ms 2024-12-09T14:47:35,279 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968; continuing. 2024-12-09T14:47:35,279 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 so closing down 2024-12-09T14:47:35,279 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:35,280 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733755650968.temp 2024-12-09T14:47:35,302 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000003-wal.1733755650968.temp 2024-12-09T14:47:35,303 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:35,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741923_1102 (size=800) 2024-12-09T14:47:35,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741923_1102 (size=800) 2024-12-09T14:47:35,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741923_1102 (size=800) 2024-12-09T14:47:35,310 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000003-wal.1733755650968.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:35,311 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000003-wal.1733755650968.temp to hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000008 2024-12-09T14:47:35,312 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 35 ms; skipped=3; WAL=hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T14:47:35,312 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968, journal: Splitting hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968, size=0 (0bytes) at 1733755651201Finishing writing output for hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 so closing down at 1733755655279 (+4078 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000003-wal.1733755650968.temp at 1733755655302 (+23 ms)3 split writer threads finished at 1733755655303 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000003-wal.1733755650968.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733755655310 (+7 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000003-wal.1733755650968.temp to hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000008 at 1733755655312 (+2 ms)Processed 5 edits across 1 Regions in 35 ms; skipped=3; WAL=hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968, size=0, length=0, corrupted=false, cancelled=false at 1733755655312 2024-12-09T14:47:35,313 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 to hdfs://localhost:35639/hbase/oldWALs/wal.1733755650968 2024-12-09T14:47:35,314 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000008 2024-12-09T14:47:35,314 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:35,316 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:35,327 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755655316, exclude list is [], retry=0 2024-12-09T14:47:35,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:35,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:35,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:35,353 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755655316 2024-12-09T14:47:35,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:35,353 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b10cb8ed5eae23f4d66ef929dc5d1884, NAME => 'testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:35,354 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:35,354 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,354 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,355 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,356 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b10cb8ed5eae23f4d66ef929dc5d1884 columnFamilyName a 2024-12-09T14:47:35,356 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,362 DEBUG [StoreFileOpener-b10cb8ed5eae23f4d66ef929dc5d1884-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:47:35,362 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_ 2024-12-09T14:47:35,366 DEBUG [StoreFileOpener-b10cb8ed5eae23f4d66ef929dc5d1884-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 928312febd804a5e875ddcdcd289eac3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:47:35,366 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/928312febd804a5e875ddcdcd289eac3_SeqId_4_ 2024-12-09T14:47:35,370 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/9d07f04d3edb4c4bbd9b3e602f8a4f92 2024-12-09T14:47:35,374 DEBUG [StoreFileOpener-b10cb8ed5eae23f4d66ef929dc5d1884-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-09T14:47:35,374 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_ 2024-12-09T14:47:35,378 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/d2f455471873437c86489a23ab2affba 2024-12-09T14:47:35,378 WARN [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4ed10183 2024-12-09T14:47:35,378 WARN [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/928312febd804a5e875ddcdcd289eac3_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4ed10183 2024-12-09T14:47:35,378 WARN [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/9d07f04d3edb4c4bbd9b3e602f8a4f92 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4ed10183 2024-12-09T14:47:35,378 WARN [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4ed10183 2024-12-09T14:47:35,378 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_, hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/928312febd804a5e875ddcdcd289eac3_SeqId_4_, hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/9d07f04d3edb4c4bbd9b3e602f8a4f92, hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_] to archive 2024-12-09T14:47:35,379 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T14:47:35,381 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_ to hdfs://localhost:35639/hbase/archive/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_ 2024-12-09T14:47:35,382 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/928312febd804a5e875ddcdcd289eac3_SeqId_4_ to hdfs://localhost:35639/hbase/archive/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/928312febd804a5e875ddcdcd289eac3_SeqId_4_ 2024-12-09T14:47:35,383 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/9d07f04d3edb4c4bbd9b3e602f8a4f92 to hdfs://localhost:35639/hbase/archive/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/9d07f04d3edb4c4bbd9b3e602f8a4f92 2024-12-09T14:47:35,384 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_ to hdfs://localhost:35639/hbase/archive/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/a/cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_ 2024-12-09T14:47:35,384 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(327): Store=b10cb8ed5eae23f4d66ef929dc5d1884/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,384 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,385 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b10cb8ed5eae23f4d66ef929dc5d1884 columnFamilyName b 2024-12-09T14:47:35,385 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,385 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(327): Store=b10cb8ed5eae23f4d66ef929dc5d1884/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,385 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,386 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b10cb8ed5eae23f4d66ef929dc5d1884 columnFamilyName c 2024-12-09T14:47:35,386 DEBUG [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,386 INFO [StoreOpener-b10cb8ed5eae23f4d66ef929dc5d1884-1 {}] regionserver.HStore(327): Store=b10cb8ed5eae23f4d66ef929dc5d1884/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,387 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,387 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,388 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,389 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000008 2024-12-09T14:47:35,409 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T14:47:35,413 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:35,415 DEBUG [Time-limited test {}] regionserver.HRegion(5836): b10cb8ed5eae23f4d66ef929dc5d1884 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "b10cb8ed5eae23f4d66ef929dc5d1884" family_name: "a" compaction_input: "9d07f04d3edb4c4bbd9b3e602f8a4f92" compaction_input: "cffdb1be6eb24419ac3ce187a9c9f982_SeqId_4_" compaction_input: "11bd9ca6763f48018b57a9ed07c7fe82_SeqId_4_" compaction_input: "928312febd804a5e875ddcdcd289eac3_SeqId_4_" compaction_output: "d2f455471873437c86489a23ab2affba" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-09T14:47:35,415 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-09T14:47:35,415 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000008 2024-12-09T14:47:35,416 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/0000000000000000008 2024-12-09T14:47:35,417 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,417 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,418 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:35,419 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b10cb8ed5eae23f4d66ef929dc5d1884 2024-12-09T14:47:35,422 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testCompactedBulkLoadedFiles/b10cb8ed5eae23f4d66ef929dc5d1884/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-09T14:47:35,423 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b10cb8ed5eae23f4d66ef929dc5d1884; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70904527, jitterRate=0.056559786200523376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:35,423 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b10cb8ed5eae23f4d66ef929dc5d1884: Writing region info on filesystem at 1733755655354Initializing all the Stores at 1733755655355 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655355Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655355Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655355Cleaning up temporary data from old regions at 1733755655417 (+62 ms)Region opened successfully at 1733755655423 (+6 ms) 2024-12-09T14:47:35,426 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b10cb8ed5eae23f4d66ef929dc5d1884, disabling compactions & flushes 2024-12-09T14:47:35,426 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. 2024-12-09T14:47:35,426 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. 2024-12-09T14:47:35,426 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. after waiting 0 ms 2024-12-09T14:47:35,426 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. 2024-12-09T14:47:35,427 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733755650892.b10cb8ed5eae23f4d66ef929dc5d1884. 2024-12-09T14:47:35,427 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b10cb8ed5eae23f4d66ef929dc5d1884: Waiting for close lock at 1733755655426Disabling compacts and flushes for region at 1733755655426Disabling writes for close at 1733755655426Writing region close event to WAL at 1733755655427 (+1 ms)Closed at 1733755655427 2024-12-09T14:47:35,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741924_1103 (size=175) 2024-12-09T14:47:35,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741924_1103 (size=175) 2024-12-09T14:47:35,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741924_1103 (size=175) 2024-12-09T14:47:35,432 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:35,432 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733755655316) 2024-12-09T14:47:35,453 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=443 (was 429) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35639 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:35639 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-733222152_22 at /127.0.0.1:47884 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-733222152_22 at /127.0.0.1:50218 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1344 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=176 (was 183), ProcessCount=11 (was 11), AvailableMemoryMB=7556 (was 7595) 2024-12-09T14:47:35,453 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-12-09T14:47:35,470 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1344, MaxFileDescriptor=1048576, SystemLoadAverage=176, ProcessCount=11, AvailableMemoryMB=7555 2024-12-09T14:47:35,470 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1344 is superior to 1024 2024-12-09T14:47:35,488 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:35,490 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T14:47:35,491 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T14:47:35,493 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-65040580, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/hregion-65040580, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:35,508 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-65040580/hregion-65040580.1733755655494, exclude list is [], retry=0 2024-12-09T14:47:35,511 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:35,511 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:35,511 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:35,540 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-65040580/hregion-65040580.1733755655494 2024-12-09T14:47:35,540 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:35,541 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a84bea0087528ae40e092a56b33f48f5, NAME => 'testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35639/hbase 2024-12-09T14:47:35,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741926_1105 (size=67) 2024-12-09T14:47:35,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741926_1105 (size=67) 2024-12-09T14:47:35,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741926_1105 (size=67) 2024-12-09T14:47:35,551 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:35,553 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,554 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName a 2024-12-09T14:47:35,554 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,554 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,554 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,556 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName b 2024-12-09T14:47:35,556 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,556 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,556 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,557 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName c 2024-12-09T14:47:35,557 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,558 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,558 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,558 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,559 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,560 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,560 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,560 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:35,561 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,563 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T14:47:35,564 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a84bea0087528ae40e092a56b33f48f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65434663, jitterRate=-0.02494753897190094}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:35,564 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a84bea0087528ae40e092a56b33f48f5: Writing region info on filesystem at 1733755655551Initializing all the Stores at 1733755655552 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655552Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655552Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655552Cleaning up temporary data from old regions at 1733755655560 (+8 ms)Region opened successfully at 1733755655564 (+4 ms) 2024-12-09T14:47:35,564 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a84bea0087528ae40e092a56b33f48f5, disabling compactions & flushes 2024-12-09T14:47:35,564 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:35,564 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:35,564 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. after waiting 0 ms 2024-12-09T14:47:35,564 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:35,565 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:35,565 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a84bea0087528ae40e092a56b33f48f5: Waiting for close lock at 1733755655564Disabling compacts and flushes for region at 1733755655564Disabling writes for close at 1733755655564Writing region close event to WAL at 1733755655564Closed at 1733755655564 2024-12-09T14:47:35,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741925_1104 (size=175) 2024-12-09T14:47:35,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741925_1104 (size=175) 2024-12-09T14:47:35,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741925_1104 (size=175) 2024-12-09T14:47:35,569 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:35,569 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-65040580:(num 1733755655494) 2024-12-09T14:47:35,569 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:35,571 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:35,581 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571, exclude list is [], retry=0 2024-12-09T14:47:35,584 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:35,584 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:35,584 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:35,607 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571 2024-12-09T14:47:35,607 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:35,607 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a84bea0087528ae40e092a56b33f48f5, NAME => 'testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:35,607 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:35,607 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,607 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,609 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,610 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName a 2024-12-09T14:47:35,610 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,610 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,611 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,611 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName b 2024-12-09T14:47:35,611 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,612 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,612 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,613 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName c 2024-12-09T14:47:35,613 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,613 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,614 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,614 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,615 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,616 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,616 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,617 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:35,618 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,620 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a84bea0087528ae40e092a56b33f48f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61640542, jitterRate=-0.0814843475818634}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:35,620 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a84bea0087528ae40e092a56b33f48f5: Writing region info on filesystem at 1733755655607Initializing all the Stores at 1733755655608 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655608Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655609 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655609Cleaning up temporary data from old regions at 1733755655616 (+7 ms)Region opened successfully at 1733755655620 (+4 ms) 2024-12-09T14:47:35,627 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a84bea0087528ae40e092a56b33f48f5 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-09T14:47:35,641 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/a/efb7d8fb69bb4e439be9619bd9caf71e is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733755655620/Put/seqid=0 2024-12-09T14:47:35,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741928_1107 (size=5958) 2024-12-09T14:47:35,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741928_1107 (size=5958) 2024-12-09T14:47:35,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741928_1107 (size=5958) 2024-12-09T14:47:35,647 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/a/efb7d8fb69bb4e439be9619bd9caf71e 2024-12-09T14:47:35,652 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/a/efb7d8fb69bb4e439be9619bd9caf71e as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/a/efb7d8fb69bb4e439be9619bd9caf71e 2024-12-09T14:47:35,657 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/a/efb7d8fb69bb4e439be9619bd9caf71e, entries=10, sequenceid=13, filesize=5.8 K 2024-12-09T14:47:35,658 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a84bea0087528ae40e092a56b33f48f5 in 30ms, sequenceid=13, compaction requested=false 2024-12-09T14:47:35,658 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a84bea0087528ae40e092a56b33f48f5: 2024-12-09T14:47:35,673 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a84bea0087528ae40e092a56b33f48f5, disabling compactions & flushes 2024-12-09T14:47:35,673 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:35,673 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:35,673 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. after waiting 0 ms 2024-12-09T14:47:35,673 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:35,677 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:35,677 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:35,677 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a84bea0087528ae40e092a56b33f48f5: Waiting for close lock at 1733755655673Disabling compacts and flushes for region at 1733755655673Disabling writes for close at 1733755655673Writing region close event to WAL at 1733755655676 (+3 ms)Closed at 1733755655677 (+1 ms) 2024-12-09T14:47:35,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741927_1106 (size=6243) 2024-12-09T14:47:35,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741927_1106 (size=6243) 2024-12-09T14:47:35,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741927_1106 (size=6243) 2024-12-09T14:47:35,697 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571, size=6.1 K (6243bytes) 2024-12-09T14:47:35,697 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571 2024-12-09T14:47:35,697 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571 after 0ms 2024-12-09T14:47:35,718 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:35,718 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571 took 22ms 2024-12-09T14:47:35,720 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571 so closing down 2024-12-09T14:47:35,720 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:35,721 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733755655571.temp 2024-12-09T14:47:35,742 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000003-wal.1733755655571.temp 2024-12-09T14:47:35,743 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:35,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741929_1108 (size=5575) 2024-12-09T14:47:35,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741929_1108 (size=5575) 2024-12-09T14:47:35,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741929_1108 (size=5575) 2024-12-09T14:47:35,751 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000003-wal.1733755655571.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:35,753 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000003-wal.1733755655571.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000035 2024-12-09T14:47:35,753 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-09T14:47:35,753 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571, journal: Splitting hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571, size=6.1 K (6243bytes) at 1733755655697Finishing writing output for hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571 so closing down at 1733755655720 (+23 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000003-wal.1733755655571.temp at 1733755655742 (+22 ms)3 split writer threads finished at 1733755655743 (+1 ms)Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000003-wal.1733755655571.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733755655751 (+8 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000003-wal.1733755655571.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000035 at 1733755655753 (+2 ms)Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733755655753 2024-12-09T14:47:35,754 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655571 to hdfs://localhost:35639/hbase/oldWALs/wal.1733755655571 2024-12-09T14:47:35,755 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000035 2024-12-09T14:47:35,756 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:35,758 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:35,768 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758, exclude list is [], retry=0 2024-12-09T14:47:35,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:35,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:35,770 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:35,793 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 2024-12-09T14:47:35,793 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373),(127.0.0.1/127.0.0.1:35879:35879)] 2024-12-09T14:47:35,793 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a84bea0087528ae40e092a56b33f48f5, NAME => 'testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5.', STARTKEY => '', ENDKEY => ''} 2024-12-09T14:47:35,793 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:35,793 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,793 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,794 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,795 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName a 2024-12-09T14:47:35,795 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,800 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/a/efb7d8fb69bb4e439be9619bd9caf71e 2024-12-09T14:47:35,800 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,800 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,801 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName b 2024-12-09T14:47:35,801 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,802 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,802 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,802 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName c 2024-12-09T14:47:35,802 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:35,803 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:35,803 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,805 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,806 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000035 2024-12-09T14:47:35,829 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:35,830 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000035 2024-12-09T14:47:35,830 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a84bea0087528ae40e092a56b33f48f5 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-09T14:47:35,842 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/b/3d8c73a421c0484daf27e362c137a79b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733755655658/Put/seqid=0 2024-12-09T14:47:35,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741931_1110 (size=5958) 2024-12-09T14:47:35,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741931_1110 (size=5958) 2024-12-09T14:47:35,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741931_1110 (size=5958) 2024-12-09T14:47:35,851 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/b/3d8c73a421c0484daf27e362c137a79b 2024-12-09T14:47:35,868 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/c/26739463badc4087bef0a34d899f87e7 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733755655664/Put/seqid=0 2024-12-09T14:47:35,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741932_1111 (size=5958) 2024-12-09T14:47:35,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741932_1111 (size=5958) 2024-12-09T14:47:35,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741932_1111 (size=5958) 2024-12-09T14:47:35,874 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/c/26739463badc4087bef0a34d899f87e7 2024-12-09T14:47:35,880 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/b/3d8c73a421c0484daf27e362c137a79b as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/b/3d8c73a421c0484daf27e362c137a79b 2024-12-09T14:47:35,886 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/b/3d8c73a421c0484daf27e362c137a79b, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T14:47:35,887 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/c/26739463badc4087bef0a34d899f87e7 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/c/26739463badc4087bef0a34d899f87e7 2024-12-09T14:47:35,892 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/c/26739463badc4087bef0a34d899f87e7, entries=10, sequenceid=35, filesize=5.8 K 2024-12-09T14:47:35,892 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for a84bea0087528ae40e092a56b33f48f5 in 62ms, sequenceid=35, compaction requested=false; wal=null 2024-12-09T14:47:35,893 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000035 2024-12-09T14:47:35,894 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,894 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,894 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:35,896 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:35,898 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-09T14:47:35,899 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a84bea0087528ae40e092a56b33f48f5; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70496163, jitterRate=0.05047468841075897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:35,899 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a84bea0087528ae40e092a56b33f48f5: Writing region info on filesystem at 1733755655793Initializing all the Stores at 1733755655794 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655794Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655794Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755655794Obtaining lock to block concurrent updates at 1733755655830 (+36 ms)Preparing flush snapshotting stores in a84bea0087528ae40e092a56b33f48f5 at 1733755655830Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733755655830Flushing stores of testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. at 1733755655830Flushing a84bea0087528ae40e092a56b33f48f5/b: creating writer at 1733755655830Flushing a84bea0087528ae40e092a56b33f48f5/b: appending metadata at 1733755655841 (+11 ms)Flushing a84bea0087528ae40e092a56b33f48f5/b: closing flushed file at 1733755655842 (+1 ms)Flushing a84bea0087528ae40e092a56b33f48f5/c: creating writer at 1733755655855 (+13 ms)Flushing a84bea0087528ae40e092a56b33f48f5/c: appending metadata at 1733755655867 (+12 ms)Flushing a84bea0087528ae40e092a56b33f48f5/c: closing flushed file at 1733755655867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26809d72: reopening flushed file at 1733755655879 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30dd54cb: reopening flushed file at 1733755655886 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for a84bea0087528ae40e092a56b33f48f5 in 62ms, sequenceid=35, compaction requested=false; wal=null at 1733755655892 (+6 ms)Cleaning up temporary data from old regions at 1733755655894 (+2 ms)Region opened successfully at 1733755655899 (+5 ms) 2024-12-09T14:47:35,954 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758, size=0 (0bytes) 2024-12-09T14:47:35,954 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 might be still open, length is 0 2024-12-09T14:47:35,954 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 2024-12-09T14:47:35,954 WARN [IPC Server handler 0 on default port 35639 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-09T14:47:35,954 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 after 0ms 2024-12-09T14:47:36,187 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:50258 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37163:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50258 dst: /127.0.0.1:37163 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37163 remote=/127.0.0.1:50258]. Total timeout mills is 60000, 59735 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:36,188 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:45908 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:46775:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45908 dst: /127.0.0.1:46775 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:36,188 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1200772549_22 at /127.0.0.1:47964 [Receiving block BP-2135414708-172.17.0.2-1733755617276:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33317:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47964 dst: /127.0.0.1:33317 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:36,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741930_1112 (size=5567) 2024-12-09T14:47:36,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741930_1112 (size=5567) 2024-12-09T14:47:39,956 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 after 4001ms 2024-12-09T14:47:39,988 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:39,988 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 took 4034ms 2024-12-09T14:47:39,990 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758; continuing. 2024-12-09T14:47:39,990 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 so closing down 2024-12-09T14:47:39,990 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-09T14:47:39,992 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733755655758.temp 2024-12-09T14:47:40,014 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000037-wal.1733755655758.temp 2024-12-09T14:47:40,014 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-09T14:47:40,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741933_1113 (size=5575) 2024-12-09T14:47:40,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741933_1113 (size=5575) 2024-12-09T14:47:40,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741933_1113 (size=5575) 2024-12-09T14:47:40,020 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000037-wal.1733755655758.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-09T14:47:40,021 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000037-wal.1733755655758.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000066 2024-12-09T14:47:40,021 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758, size=0, length=0, corrupted=false, cancelled=false 2024-12-09T14:47:40,021 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758, journal: Splitting hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758, size=0 (0bytes) at 1733755655954Finishing writing output for hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 so closing down at 1733755659990 (+4036 ms)Creating recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000037-wal.1733755655758.temp at 1733755660014 (+24 ms)3 split writer threads finished at 1733755660014Closed recovered edits writer path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000037-wal.1733755655758.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733755660020 (+6 ms)Rename recovered edits hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000037-wal.1733755655758.temp to hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000066 at 1733755660021 (+1 ms)Processed 30 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758, size=0, length=0, corrupted=false, cancelled=false at 1733755660021 2024-12-09T14:47:40,023 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 to hdfs://localhost:35639/hbase/oldWALs/wal.1733755655758 2024-12-09T14:47:40,024 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000066 2024-12-09T14:47:40,024 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-09T14:47:40,025 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:35639/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487, archiveDir=hdfs://localhost:35639/hbase/oldWALs, maxLogs=32 2024-12-09T14:47:40,037 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755660026, exclude list is [], retry=0 2024-12-09T14:47:40,040 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33317,DS-89e8a131-3dbd-4de8-8f39-b1d9e50471fc,DISK] 2024-12-09T14:47:40,040 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37163,DS-881664bb-1345-45a8-9a41-c28a0e496621,DISK] 2024-12-09T14:47:40,040 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46775,DS-c0e9d82e-4339-49b1-9400-e7877e2718e9,DISK] 2024-12-09T14:47:40,061 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755660026 2024-12-09T14:47:40,061 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35879:35879),(127.0.0.1/127.0.0.1:46429:46429),(127.0.0.1/127.0.0.1:34373:34373)] 2024-12-09T14:47:40,061 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T14:47:40,063 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:40,063 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName a 2024-12-09T14:47:40,063 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:40,068 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/a/efb7d8fb69bb4e439be9619bd9caf71e 2024-12-09T14:47:40,068 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:40,068 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:40,068 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName b 2024-12-09T14:47:40,069 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:40,073 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/b/3d8c73a421c0484daf27e362c137a79b 2024-12-09T14:47:40,074 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:40,074 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:40,075 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a84bea0087528ae40e092a56b33f48f5 columnFamilyName c 2024-12-09T14:47:40,075 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T14:47:40,080 DEBUG [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/c/26739463badc4087bef0a34d899f87e7 2024-12-09T14:47:40,081 INFO [StoreOpener-a84bea0087528ae40e092a56b33f48f5-1 {}] regionserver.HStore(327): Store=a84bea0087528ae40e092a56b33f48f5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T14:47:40,081 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:40,082 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:40,083 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:40,084 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000066 2024-12-09T14:47:40,108 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-09T14:47:40,111 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000066 2024-12-09T14:47:40,111 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a84bea0087528ae40e092a56b33f48f5 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-09T14:47:40,124 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/a/93a4885d0cda4072892a49d104e626df is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733755655906/Put/seqid=0 2024-12-09T14:47:40,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741935_1115 (size=5958) 2024-12-09T14:47:40,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741935_1115 (size=5958) 2024-12-09T14:47:40,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741935_1115 (size=5958) 2024-12-09T14:47:40,130 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/a/93a4885d0cda4072892a49d104e626df 2024-12-09T14:47:40,146 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/b/7de9653e745342e68f82bcf4b6ed4a13 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733755655911/Put/seqid=0 2024-12-09T14:47:40,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741936_1116 (size=5958) 2024-12-09T14:47:40,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741936_1116 (size=5958) 2024-12-09T14:47:40,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741936_1116 (size=5958) 2024-12-09T14:47:40,153 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/b/7de9653e745342e68f82bcf4b6ed4a13 2024-12-09T14:47:40,169 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/c/dd9620bdf3a74ef3991008e6651af8b7 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733755655918/Put/seqid=0 2024-12-09T14:47:40,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741937_1117 (size=5958) 2024-12-09T14:47:40,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741937_1117 (size=5958) 2024-12-09T14:47:40,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741937_1117 (size=5958) 2024-12-09T14:47:40,175 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/c/dd9620bdf3a74ef3991008e6651af8b7 2024-12-09T14:47:40,180 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/a/93a4885d0cda4072892a49d104e626df as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/a/93a4885d0cda4072892a49d104e626df 2024-12-09T14:47:40,184 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/a/93a4885d0cda4072892a49d104e626df, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T14:47:40,185 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/b/7de9653e745342e68f82bcf4b6ed4a13 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/b/7de9653e745342e68f82bcf4b6ed4a13 2024-12-09T14:47:40,190 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/b/7de9653e745342e68f82bcf4b6ed4a13, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T14:47:40,191 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/.tmp/c/dd9620bdf3a74ef3991008e6651af8b7 as hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/c/dd9620bdf3a74ef3991008e6651af8b7 2024-12-09T14:47:40,195 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/c/dd9620bdf3a74ef3991008e6651af8b7, entries=10, sequenceid=66, filesize=5.8 K 2024-12-09T14:47:40,195 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for a84bea0087528ae40e092a56b33f48f5 in 84ms, sequenceid=66, compaction requested=false; wal=null 2024-12-09T14:47:40,196 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/0000000000000000066 2024-12-09T14:47:40,197 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:40,197 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:40,197 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T14:47:40,198 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a84bea0087528ae40e092a56b33f48f5 2024-12-09T14:47:40,200 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/hbase/data/default/testReplayEditsWrittenViaHRegion/a84bea0087528ae40e092a56b33f48f5/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-09T14:47:40,201 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a84bea0087528ae40e092a56b33f48f5; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62278642, jitterRate=-0.07197591662406921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T14:47:40,201 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a84bea0087528ae40e092a56b33f48f5: Writing region info on filesystem at 1733755660061Initializing all the Stores at 1733755660062 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755660062Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755660062Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733755660062Obtaining lock to block concurrent updates at 1733755660111 (+49 ms)Preparing flush snapshotting stores in a84bea0087528ae40e092a56b33f48f5 at 1733755660111Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733755660112 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. at 1733755660112Flushing a84bea0087528ae40e092a56b33f48f5/a: creating writer at 1733755660112Flushing a84bea0087528ae40e092a56b33f48f5/a: appending metadata at 1733755660123 (+11 ms)Flushing a84bea0087528ae40e092a56b33f48f5/a: closing flushed file at 1733755660123Flushing a84bea0087528ae40e092a56b33f48f5/b: creating writer at 1733755660134 (+11 ms)Flushing a84bea0087528ae40e092a56b33f48f5/b: appending metadata at 1733755660146 (+12 ms)Flushing a84bea0087528ae40e092a56b33f48f5/b: closing flushed file at 1733755660146Flushing a84bea0087528ae40e092a56b33f48f5/c: creating writer at 1733755660157 (+11 ms)Flushing a84bea0087528ae40e092a56b33f48f5/c: appending metadata at 1733755660168 (+11 ms)Flushing a84bea0087528ae40e092a56b33f48f5/c: closing flushed file at 1733755660168Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fa570b4: reopening flushed file at 1733755660179 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e918670: reopening flushed file at 1733755660184 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13054e3f: reopening flushed file at 1733755660190 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for a84bea0087528ae40e092a56b33f48f5 in 84ms, sequenceid=66, compaction requested=false; wal=null at 1733755660195 (+5 ms)Cleaning up temporary data from old regions at 1733755660197 (+2 ms)Region opened successfully at 1733755660201 (+4 ms) 2024-12-09T14:47:40,212 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a84bea0087528ae40e092a56b33f48f5, disabling compactions & flushes 2024-12-09T14:47:40,212 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:40,213 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:40,213 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. after waiting 0 ms 2024-12-09T14:47:40,213 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:40,214 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733755655489.a84bea0087528ae40e092a56b33f48f5. 2024-12-09T14:47:40,214 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a84bea0087528ae40e092a56b33f48f5: Waiting for close lock at 1733755660212Disabling compacts and flushes for region at 1733755660212Disabling writes for close at 1733755660213 (+1 ms)Writing region close event to WAL at 1733755660214 (+1 ms)Closed at 1733755660214 2024-12-09T14:47:40,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741934_1114 (size=175) 2024-12-09T14:47:40,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741934_1114 (size=175) 2024-12-09T14:47:40,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741934_1114 (size=175) 2024-12-09T14:47:40,270 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-09T14:47:40,270 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733755660026) 2024-12-09T14:47:40,282 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=448 (was 443) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:35639 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_6509905_22 at /127.0.0.1:50294 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_6509905_22 at /127.0.0.1:45926 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35639 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_6509905_22 at /127.0.0.1:48000 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1344) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=162 (was 176), ProcessCount=11 (was 11), AvailableMemoryMB=7524 (was 7555) 2024-12-09T14:47:40,282 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-12-09T14:47:40,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T14:47:40,282 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T14:47:40,282 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:47:40,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:47:40,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:47:40,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T14:47:40,283 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T14:47:40,283 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=482707301, stopped=false 2024-12-09T14:47:40,283 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2491923345a5,40599,1733755621019 2024-12-09T14:47:40,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:47:40,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:47:40,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T14:47:40,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:40,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:40,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:40,332 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T14:47:40,333 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:47:40,333 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T14:47:40,333 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:47:40,333 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:47:40,333 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T14:47:40,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:47:40,334 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2491923345a5,43689,1733755621826' ***** 2024-12-09T14:47:40,334 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T14:47:40,334 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2491923345a5,44703,1733755621870' ***** 2024-12-09T14:47:40,334 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T14:47:40,335 INFO [RS:1;2491923345a5:43689 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:47:40,335 INFO [RS:1;2491923345a5:43689 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T14:47:40,335 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:47:40,335 INFO [RS:1;2491923345a5:43689 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T14:47:40,335 INFO [RS:2;2491923345a5:44703 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T14:47:40,335 INFO [RS:2;2491923345a5:44703 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T14:47:40,335 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T14:47:40,335 INFO [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(959): stopping server 2491923345a5,43689,1733755621826 2024-12-09T14:47:40,335 INFO [RS:2;2491923345a5:44703 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T14:47:40,335 INFO [RS:1;2491923345a5:43689 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:47:40,336 INFO [RS:1;2491923345a5:43689 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2491923345a5:43689. 2024-12-09T14:47:40,336 DEBUG [RS:1;2491923345a5:43689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:47:40,336 DEBUG [RS:1;2491923345a5:43689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:47:40,336 INFO [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(3091): Received CLOSE for 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:40,336 INFO [RS:1;2491923345a5:43689 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:47:40,336 INFO [RS:1;2491923345a5:43689 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:47:40,336 INFO [RS:1;2491923345a5:43689 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:47:40,336 INFO [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(959): stopping server 2491923345a5,44703,1733755621870 2024-12-09T14:47:40,336 INFO [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T14:47:40,336 INFO [RS:2;2491923345a5:44703 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:47:40,336 INFO [RS:2;2491923345a5:44703 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2491923345a5:44703. 2024-12-09T14:47:40,337 DEBUG [RS:2;2491923345a5:44703 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T14:47:40,337 DEBUG [RS:2;2491923345a5:44703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:47:40,337 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 10975ad07e6cf31ab36e8c23dc0846f3, disabling compactions & flushes 2024-12-09T14:47:40,337 INFO [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T14:47:40,337 INFO [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T14:47:40,337 DEBUG [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(1325): Online Regions={10975ad07e6cf31ab36e8c23dc0846f3=testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3.} 2024-12-09T14:47:40,337 DEBUG [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T14:47:40,337 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:40,337 DEBUG [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(1351): Waiting on 10975ad07e6cf31ab36e8c23dc0846f3 2024-12-09T14:47:40,337 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T14:47:40,337 DEBUG [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T14:47:40,337 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:40,337 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T14:47:40,337 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. after waiting 0 ms 2024-12-09T14:47:40,337 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T14:47:40,337 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:40,337 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T14:47:40,337 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T14:47:40,337 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-09T14:47:40,339 INFO [regionserver/2491923345a5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:47:40,339 INFO [regionserver/2491923345a5:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:47:40,342 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/10975ad07e6cf31ab36e8c23dc0846f3/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-09T14:47:40,343 INFO [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:40,343 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 10975ad07e6cf31ab36e8c23dc0846f3: Waiting for close lock at 1733755660336Running coprocessor pre-close hooks at 1733755660336Disabling compacts and flushes for region at 1733755660336Disabling writes for close at 1733755660337 (+1 ms)Writing region close event to WAL at 1733755660339 (+2 ms)Running coprocessor post-close hooks at 1733755660343 (+4 ms)Closed at 1733755660343 2024-12-09T14:47:40,343 DEBUG [RS_CLOSE_REGION-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3. 2024-12-09T14:47:40,357 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/.tmp/info/e4c46cb6fbdb415eb13a754a6fc63eb4 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733755636730.10975ad07e6cf31ab36e8c23dc0846f3./info:regioninfo/1733755639854/Put/seqid=0 2024-12-09T14:47:40,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741938_1118 (size=8243) 2024-12-09T14:47:40,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741938_1118 (size=8243) 2024-12-09T14:47:40,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741938_1118 (size=8243) 2024-12-09T14:47:40,364 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/.tmp/info/e4c46cb6fbdb415eb13a754a6fc63eb4 2024-12-09T14:47:40,382 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/.tmp/ns/3ec0cde42c39495394ebcdd7e0cf9215 is 43, key is default/ns:d/1733755623990/Put/seqid=0 2024-12-09T14:47:40,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741939_1119 (size=5153) 2024-12-09T14:47:40,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741939_1119 (size=5153) 2024-12-09T14:47:40,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741939_1119 (size=5153) 2024-12-09T14:47:40,389 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/.tmp/ns/3ec0cde42c39495394ebcdd7e0cf9215 2024-12-09T14:47:40,409 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/.tmp/table/87415dbd10574bb4a049bb5c6d927221 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733755637145/Put/seqid=0 2024-12-09T14:47:40,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741940_1120 (size=5431) 2024-12-09T14:47:40,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741940_1120 (size=5431) 2024-12-09T14:47:40,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741940_1120 (size=5431) 2024-12-09T14:47:40,415 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/.tmp/table/87415dbd10574bb4a049bb5c6d927221 2024-12-09T14:47:40,421 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/.tmp/info/e4c46cb6fbdb415eb13a754a6fc63eb4 as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/info/e4c46cb6fbdb415eb13a754a6fc63eb4 2024-12-09T14:47:40,428 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/info/e4c46cb6fbdb415eb13a754a6fc63eb4, entries=18, sequenceid=21, filesize=8.0 K 2024-12-09T14:47:40,429 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/.tmp/ns/3ec0cde42c39495394ebcdd7e0cf9215 as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/ns/3ec0cde42c39495394ebcdd7e0cf9215 2024-12-09T14:47:40,434 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/ns/3ec0cde42c39495394ebcdd7e0cf9215, entries=2, sequenceid=21, filesize=5.0 K 2024-12-09T14:47:40,435 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/.tmp/table/87415dbd10574bb4a049bb5c6d927221 as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/table/87415dbd10574bb4a049bb5c6d927221 2024-12-09T14:47:40,439 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/table/87415dbd10574bb4a049bb5c6d927221, entries=2, sequenceid=21, filesize=5.3 K 2024-12-09T14:47:40,440 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=21, compaction requested=false 2024-12-09T14:47:40,444 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-09T14:47:40,445 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T14:47:40,445 INFO [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T14:47:40,445 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733755660337Running coprocessor pre-close hooks at 1733755660337Disabling compacts and flushes for region at 1733755660337Disabling writes for close at 1733755660337Obtaining lock to block concurrent updates at 1733755660337Preparing flush snapshotting stores in 1588230740 at 1733755660337Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733755660338 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733755660339 (+1 ms)Flushing 1588230740/info: creating writer at 1733755660339Flushing 1588230740/info: appending metadata at 1733755660356 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733755660357 (+1 ms)Flushing 1588230740/ns: creating writer at 1733755660369 (+12 ms)Flushing 1588230740/ns: appending metadata at 1733755660381 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733755660381Flushing 1588230740/table: creating writer at 1733755660394 (+13 ms)Flushing 1588230740/table: appending metadata at 1733755660408 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733755660408Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64dc29d3: reopening flushed file at 1733755660420 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1526de77: reopening flushed file at 1733755660428 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56eca786: reopening flushed file at 1733755660434 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=21, compaction requested=false at 1733755660440 (+6 ms)Writing region close event to WAL at 1733755660441 (+1 ms)Running coprocessor post-close hooks at 1733755660445 (+4 ms)Closed at 1733755660445 2024-12-09T14:47:40,445 DEBUG [RS_CLOSE_META-regionserver/2491923345a5:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T14:47:40,537 INFO [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(976): stopping server 2491923345a5,44703,1733755621870; all regions closed. 2024-12-09T14:47:40,537 INFO [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(976): stopping server 2491923345a5,43689,1733755621826; all regions closed. 2024-12-09T14:47:40,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741833_1009 (size=1032) 2024-12-09T14:47:40,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741833_1009 (size=1032) 2024-12-09T14:47:40,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741836_1012 (size=9487) 2024-12-09T14:47:40,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741836_1012 (size=9487) 2024-12-09T14:47:40,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741833_1009 (size=1032) 2024-12-09T14:47:40,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741836_1012 (size=9487) 2024-12-09T14:47:40,546 DEBUG [RS:2;2491923345a5:44703 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/oldWALs 2024-12-09T14:47:40,547 INFO [RS:2;2491923345a5:44703 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C44703%2C1733755621870:(num 1733755623457) 2024-12-09T14:47:40,547 DEBUG [RS:1;2491923345a5:43689 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/oldWALs 2024-12-09T14:47:40,547 DEBUG [RS:2;2491923345a5:44703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:47:40,547 INFO [RS:2;2491923345a5:44703 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:47:40,547 INFO [RS:1;2491923345a5:43689 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C43689%2C1733755621826.meta:.meta(num 1733755623805) 2024-12-09T14:47:40,547 INFO [RS:2;2491923345a5:44703 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:47:40,547 INFO [RS:2;2491923345a5:44703 {}] hbase.ChoreService(370): Chore service for: regionserver/2491923345a5:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T14:47:40,547 INFO [RS:2;2491923345a5:44703 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T14:47:40,547 INFO [RS:2;2491923345a5:44703 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T14:47:40,547 INFO [regionserver/2491923345a5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:47:40,547 INFO [RS:2;2491923345a5:44703 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T14:47:40,547 INFO [RS:2;2491923345a5:44703 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:47:40,547 INFO [RS:2;2491923345a5:44703 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44703 2024-12-09T14:47:40,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741834_1010 (size=2764) 2024-12-09T14:47:40,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741834_1010 (size=2764) 2024-12-09T14:47:40,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741834_1010 (size=2764) 2024-12-09T14:47:40,551 DEBUG [RS:1;2491923345a5:43689 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/oldWALs 2024-12-09T14:47:40,551 INFO [RS:1;2491923345a5:43689 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2491923345a5%2C43689%2C1733755621826:(num 1733755623457) 2024-12-09T14:47:40,551 DEBUG [RS:1;2491923345a5:43689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T14:47:40,551 INFO [RS:1;2491923345a5:43689 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T14:47:40,551 INFO [RS:1;2491923345a5:43689 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:47:40,552 INFO [RS:1;2491923345a5:43689 {}] hbase.ChoreService(370): Chore service for: regionserver/2491923345a5:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T14:47:40,552 INFO [RS:1;2491923345a5:43689 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:47:40,552 INFO [regionserver/2491923345a5:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:47:40,552 INFO [RS:1;2491923345a5:43689 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43689 2024-12-09T14:47:40,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T14:47:40,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2491923345a5,44703,1733755621870 2024-12-09T14:47:40,581 INFO [RS:2;2491923345a5:44703 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:47:40,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2491923345a5,43689,1733755621826 2024-12-09T14:47:40,634 INFO [RS:1;2491923345a5:43689 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:47:40,642 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2491923345a5,43689,1733755621826] 2024-12-09T14:47:40,658 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2491923345a5,43689,1733755621826 already deleted, retry=false 2024-12-09T14:47:40,659 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2491923345a5,43689,1733755621826 expired; onlineServers=1 2024-12-09T14:47:40,659 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2491923345a5,44703,1733755621870] 2024-12-09T14:47:40,667 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2491923345a5,44703,1733755621870 already deleted, retry=false 2024-12-09T14:47:40,667 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2491923345a5,44703,1733755621870 expired; onlineServers=0 2024-12-09T14:47:40,667 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2491923345a5,40599,1733755621019' ***** 2024-12-09T14:47:40,667 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T14:47:40,667 INFO [M:0;2491923345a5:40599 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T14:47:40,668 INFO [M:0;2491923345a5:40599 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T14:47:40,668 DEBUG [M:0;2491923345a5:40599 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T14:47:40,668 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T14:47:40,668 DEBUG [M:0;2491923345a5:40599 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T14:47:40,668 DEBUG [master/2491923345a5:0:becomeActiveMaster-HFileCleaner.small.0-1733755623021 {}] cleaner.HFileCleaner(306): Exit Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.small.0-1733755623021,5,FailOnTimeoutGroup] 2024-12-09T14:47:40,668 DEBUG [master/2491923345a5:0:becomeActiveMaster-HFileCleaner.large.0-1733755623020 {}] cleaner.HFileCleaner(306): Exit Thread[master/2491923345a5:0:becomeActiveMaster-HFileCleaner.large.0-1733755623020,5,FailOnTimeoutGroup] 2024-12-09T14:47:40,669 INFO [M:0;2491923345a5:40599 {}] hbase.ChoreService(370): Chore service for: master/2491923345a5:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T14:47:40,669 INFO [M:0;2491923345a5:40599 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T14:47:40,669 DEBUG [M:0;2491923345a5:40599 {}] master.HMaster(1795): Stopping service threads 2024-12-09T14:47:40,669 INFO [M:0;2491923345a5:40599 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T14:47:40,669 INFO [M:0;2491923345a5:40599 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T14:47:40,671 INFO [M:0;2491923345a5:40599 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T14:47:40,671 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T14:47:40,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T14:47:40,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T14:47:40,675 DEBUG [M:0;2491923345a5:40599 {}] zookeeper.ZKUtil(347): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T14:47:40,676 WARN [M:0;2491923345a5:40599 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T14:47:40,677 INFO [M:0;2491923345a5:40599 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/.lastflushedseqids 2024-12-09T14:47:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741941_1121 (size=138) 2024-12-09T14:47:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741941_1121 (size=138) 2024-12-09T14:47:40,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741941_1121 (size=138) 2024-12-09T14:47:40,693 INFO [M:0;2491923345a5:40599 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T14:47:40,693 INFO [M:0;2491923345a5:40599 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T14:47:40,694 DEBUG [M:0;2491923345a5:40599 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T14:47:40,694 INFO [M:0;2491923345a5:40599 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:47:40,694 DEBUG [M:0;2491923345a5:40599 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:47:40,694 DEBUG [M:0;2491923345a5:40599 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T14:47:40,694 DEBUG [M:0;2491923345a5:40599 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:47:40,694 INFO [M:0;2491923345a5:40599 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-12-09T14:47:40,709 DEBUG [M:0;2491923345a5:40599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d45a095fa1a547d0b4fb743b349a93ed is 82, key is hbase:meta,,1/info:regioninfo/1733755623900/Put/seqid=0 2024-12-09T14:47:40,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741942_1122 (size=5672) 2024-12-09T14:47:40,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741942_1122 (size=5672) 2024-12-09T14:47:40,714 INFO [M:0;2491923345a5:40599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d45a095fa1a547d0b4fb743b349a93ed 2024-12-09T14:47:40,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741942_1122 (size=5672) 2024-12-09T14:47:40,731 DEBUG [M:0;2491923345a5:40599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d4577768822447986f9de1bdf224af6 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733755637151/Put/seqid=0 2024-12-09T14:47:40,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741943_1123 (size=7755) 2024-12-09T14:47:40,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741943_1123 (size=7755) 2024-12-09T14:47:40,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741943_1123 (size=7755) 2024-12-09T14:47:40,738 INFO [M:0;2491923345a5:40599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d4577768822447986f9de1bdf224af6 2024-12-09T14:47:40,741 INFO [M:0;2491923345a5:40599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2d4577768822447986f9de1bdf224af6 2024-12-09T14:47:40,742 INFO [RS:2;2491923345a5:44703 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:47:40,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:47:40,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44703-0x1000b190a330003, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:47:40,742 INFO [RS:2;2491923345a5:44703 {}] regionserver.HRegionServer(1031): Exiting; stopping=2491923345a5,44703,1733755621870; zookeeper connection closed. 2024-12-09T14:47:40,742 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b53240 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b53240 2024-12-09T14:47:40,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:47:40,751 INFO [RS:1;2491923345a5:43689 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:47:40,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43689-0x1000b190a330002, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:47:40,751 INFO [RS:1;2491923345a5:43689 {}] regionserver.HRegionServer(1031): Exiting; stopping=2491923345a5,43689,1733755621826; zookeeper connection closed. 2024-12-09T14:47:40,751 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a456996 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a456996 2024-12-09T14:47:40,751 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T14:47:40,760 DEBUG [M:0;2491923345a5:40599 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4c5d830a4efb4337b9bf913cbc246926 is 69, key is 2491923345a5,43689,1733755621826/rs:state/1733755623176/Put/seqid=0 2024-12-09T14:47:40,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741944_1124 (size=5440) 2024-12-09T14:47:40,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741944_1124 (size=5440) 2024-12-09T14:47:40,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741944_1124 (size=5440) 2024-12-09T14:47:40,768 INFO [M:0;2491923345a5:40599 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4c5d830a4efb4337b9bf913cbc246926 2024-12-09T14:47:40,772 INFO [M:0;2491923345a5:40599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4c5d830a4efb4337b9bf913cbc246926 2024-12-09T14:47:40,773 DEBUG [M:0;2491923345a5:40599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d45a095fa1a547d0b4fb743b349a93ed as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d45a095fa1a547d0b4fb743b349a93ed 2024-12-09T14:47:40,777 INFO [M:0;2491923345a5:40599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d45a095fa1a547d0b4fb743b349a93ed, entries=8, sequenceid=168, filesize=5.5 K 2024-12-09T14:47:40,778 DEBUG [M:0;2491923345a5:40599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2d4577768822447986f9de1bdf224af6 as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2d4577768822447986f9de1bdf224af6 2024-12-09T14:47:40,783 INFO [M:0;2491923345a5:40599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2d4577768822447986f9de1bdf224af6 2024-12-09T14:47:40,783 INFO [M:0;2491923345a5:40599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2d4577768822447986f9de1bdf224af6, entries=17, sequenceid=168, filesize=7.6 K 2024-12-09T14:47:40,784 DEBUG [M:0;2491923345a5:40599 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4c5d830a4efb4337b9bf913cbc246926 as hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4c5d830a4efb4337b9bf913cbc246926 2024-12-09T14:47:40,789 INFO [M:0;2491923345a5:40599 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4c5d830a4efb4337b9bf913cbc246926 2024-12-09T14:47:40,790 INFO [M:0;2491923345a5:40599 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35639/user/jenkins/test-data/6084318a-0027-3599-c294-1c30217bdc7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4c5d830a4efb4337b9bf913cbc246926, entries=3, sequenceid=168, filesize=5.3 K 2024-12-09T14:47:40,790 INFO [M:0;2491923345a5:40599 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 96ms, sequenceid=168, compaction requested=false 2024-12-09T14:47:40,792 INFO [M:0;2491923345a5:40599 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T14:47:40,792 DEBUG [M:0;2491923345a5:40599 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733755660694Disabling compacts and flushes for region at 1733755660694Disabling writes for close at 1733755660694Obtaining lock to block concurrent updates at 1733755660694Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733755660694Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1733755660694Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733755660695 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733755660695Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733755660709 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733755660709Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733755660718 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733755660731 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733755660731Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733755660741 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733755660759 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733755660759Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ad267d2: reopening flushed file at 1733755660772 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@330bec91: reopening flushed file at 1733755660777 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28f4f3b0: reopening flushed file at 1733755660783 (+6 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 96ms, sequenceid=168, compaction requested=false at 1733755660790 (+7 ms)Writing region close event to WAL at 1733755660791 (+1 ms)Closed at 1733755660791 2024-12-09T14:47:40,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46775 is added to blk_1073741830_1006 (size=83606) 2024-12-09T14:47:40,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33317 is added to blk_1073741830_1006 (size=83606) 2024-12-09T14:47:40,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37163 is added to blk_1073741830_1006 (size=83606) 2024-12-09T14:47:40,795 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T14:47:40,795 INFO [M:0;2491923345a5:40599 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T14:47:40,795 INFO [M:0;2491923345a5:40599 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40599 2024-12-09T14:47:40,795 INFO [M:0;2491923345a5:40599 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T14:47:40,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:47:40,925 INFO [M:0;2491923345a5:40599 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T14:47:40,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40599-0x1000b190a330000, quorum=127.0.0.1:58381, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T14:47:40,932 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640489 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640489 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1200772549_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733755640148/wal.1733755640489 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1200772549_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T14:47:40,933 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733755632226/wal.1733755632358 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:40,934 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733755655487/wal.1733755655758 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:40,937 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755649988 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755649988 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1200772549_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733755640647/wal.1733755649988 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1200772549_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T14:47:40,937 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733755625443/wal.1733755625530 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:40,940 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650715 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650715 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1200772549_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733755650201/wal.1733755650715 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1200772549_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T14:47:40,941 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733755650891/wal.1733755650968 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T14:47:40,943 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal.1733755625266 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal.1733755625266 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1200772549_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733755624917/wal.1733755625266 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1200772549_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T14:47:40,945 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal.1733755624612 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal.1733755624612 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1200772549_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733755624192/wal.1733755624612 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1200772549_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-09T14:47:40,985 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:47:40,987 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T14:47:40,987 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T14:47:40,987 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T14:47:40,987 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/hadoop.log.dir/,STOPPED} 2024-12-09T14:47:40,989 WARN [BP-2135414708-172.17.0.2-1733755617276 heartbeating to localhost/127.0.0.1:35639 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T14:47:40,989 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T14:47:40,989 WARN [BP-2135414708-172.17.0.2-1733755617276 heartbeating to localhost/127.0.0.1:35639 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2135414708-172.17.0.2-1733755617276 (Datanode Uuid c418ed6f-3494-4a7f-b949-673db0ef1ea1) service to localhost/127.0.0.1:35639 2024-12-09T14:47:40,989 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T14:47:40,991 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data5/current/BP-2135414708-172.17.0.2-1733755617276 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:47:40,991 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data6/current/BP-2135414708-172.17.0.2-1733755617276 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:47:40,991 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T14:47:40,996 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:47:40,996 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T14:47:40,996 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T14:47:40,996 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T14:47:40,996 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/hadoop.log.dir/,STOPPED} 2024-12-09T14:47:40,997 WARN [BP-2135414708-172.17.0.2-1733755617276 heartbeating to localhost/127.0.0.1:35639 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T14:47:40,997 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T14:47:40,997 WARN [BP-2135414708-172.17.0.2-1733755617276 heartbeating to localhost/127.0.0.1:35639 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2135414708-172.17.0.2-1733755617276 (Datanode Uuid 8b469460-f07d-4e65-a333-3cf61e240627) service to localhost/127.0.0.1:35639 2024-12-09T14:47:40,997 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T14:47:40,998 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data3/current/BP-2135414708-172.17.0.2-1733755617276 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:47:40,998 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data4/current/BP-2135414708-172.17.0.2-1733755617276 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:47:40,998 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T14:47:41,000 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T14:47:41,000 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T14:47:41,001 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T14:47:41,001 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T14:47:41,001 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/hadoop.log.dir/,STOPPED} 2024-12-09T14:47:41,002 WARN [BP-2135414708-172.17.0.2-1733755617276 heartbeating to localhost/127.0.0.1:35639 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T14:47:41,002 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T14:47:41,002 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T14:47:41,002 WARN [BP-2135414708-172.17.0.2-1733755617276 heartbeating to localhost/127.0.0.1:35639 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2135414708-172.17.0.2-1733755617276 (Datanode Uuid a8fc4096-befb-456e-84a7-962edeb9a8d6) service to localhost/127.0.0.1:35639 2024-12-09T14:47:41,003 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data1/current/BP-2135414708-172.17.0.2-1733755617276 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:47:41,003 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/cluster_0df19de4-93b6-3212-3ae8-4a9f657f0ccd/data/data2/current/BP-2135414708-172.17.0.2-1733755617276 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T14:47:41,003 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T14:47:41,009 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T14:47:41,010 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T14:47:41,010 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T14:47:41,010 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T14:47:41,010 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master@2/hbase-server/target/test-data/b73b9472-9fd5-d686-80d8-58082f80b829/hadoop.log.dir/,STOPPED} 2024-12-09T14:47:41,019 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T14:47:41,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down