2024-12-01 14:29:34,406 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-01 14:29:34,419 main DEBUG Took 0.011188 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-01 14:29:34,419 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-01 14:29:34,420 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-01 14:29:34,421 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-01 14:29:34,422 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,431 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-01 14:29:34,444 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,447 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,447 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,448 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,448 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,450 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,451 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,451 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,452 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,453 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,453 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,454 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,455 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,456 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,457 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,457 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,458 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 14:29:34,459 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,459 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-01 14:29:34,461 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 14:29:34,463 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-01 14:29:34,466 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-01 14:29:34,466 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-01 14:29:34,468 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-01 14:29:34,468 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-01 14:29:34,479 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-01 14:29:34,483 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-01 14:29:34,485 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-01 14:29:34,486 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-01 14:29:34,486 main DEBUG createAppenders(={Console}) 2024-12-01 14:29:34,487 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-01 14:29:34,488 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-01 14:29:34,488 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-01 14:29:34,489 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-01 14:29:34,489 main DEBUG OutputStream closed 2024-12-01 14:29:34,490 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-01 14:29:34,490 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-01 14:29:34,490 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-01 14:29:34,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-01 14:29:34,574 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-01 14:29:34,575 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-01 14:29:34,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-01 14:29:34,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-01 14:29:34,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-01 14:29:34,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-01 14:29:34,579 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-01 14:29:34,579 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-01 14:29:34,579 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-01 14:29:34,580 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-01 14:29:34,580 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-01 14:29:34,581 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-01 14:29:34,581 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-01 14:29:34,581 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-01 14:29:34,582 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-01 14:29:34,582 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-01 14:29:34,583 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-01 14:29:34,586 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01 14:29:34,586 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-01 14:29:34,586 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-01 14:29:34,587 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-01T14:29:34,816 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726 2024-12-01 14:29:34,818 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-01 14:29:34,819 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01T14:29:34,827 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-01T14:29:34,832 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-12-01T14:29:34,851 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-01T14:29:34,891 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-01T14:29:34,891 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-01T14:29:34,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-01T14:29:34,914 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233, deleteOnExit=true 2024-12-01T14:29:34,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-01T14:29:34,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/test.cache.data in system properties and HBase conf 2024-12-01T14:29:34,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/hadoop.tmp.dir in system properties and HBase conf 2024-12-01T14:29:34,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/hadoop.log.dir in system properties and HBase conf 2024-12-01T14:29:34,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-01T14:29:34,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-01T14:29:34,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-01T14:29:34,992 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-01T14:29:35,082 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T14:29:35,086 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-01T14:29:35,086 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-01T14:29:35,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-01T14:29:35,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T14:29:35,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-01T14:29:35,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-01T14:29:35,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T14:29:35,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T14:29:35,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-01T14:29:35,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/nfs.dump.dir in system properties and HBase conf 2024-12-01T14:29:35,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/java.io.tmpdir in system properties and HBase conf 2024-12-01T14:29:35,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T14:29:35,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-01T14:29:35,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-01T14:29:36,133 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-01T14:29:36,195 INFO [Time-limited test {}] log.Log(170): Logging initialized @2419ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-01T14:29:36,258 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T14:29:36,316 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T14:29:36,337 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T14:29:36,338 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T14:29:36,339 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-01T14:29:36,354 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T14:29:36,357 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/hadoop.log.dir/,AVAILABLE} 2024-12-01T14:29:36,359 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T14:29:36,515 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/java.io.tmpdir/jetty-localhost-43115-hadoop-hdfs-3_4_1-tests_jar-_-any-4659532417846940200/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-01T14:29:36,520 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:43115} 2024-12-01T14:29:36,520 INFO [Time-limited test {}] server.Server(415): Started @2744ms 2024-12-01T14:29:37,041 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T14:29:37,047 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T14:29:37,047 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T14:29:37,048 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T14:29:37,048 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-01T14:29:37,049 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/hadoop.log.dir/,AVAILABLE} 2024-12-01T14:29:37,049 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T14:29:37,142 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/java.io.tmpdir/jetty-localhost-45129-hadoop-hdfs-3_4_1-tests_jar-_-any-4842675946642915144/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T14:29:37,143 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:45129} 2024-12-01T14:29:37,143 INFO [Time-limited test {}] server.Server(415): Started @3367ms 2024-12-01T14:29:37,186 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T14:29:37,290 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T14:29:37,296 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T14:29:37,300 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T14:29:37,301 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T14:29:37,301 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T14:29:37,303 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/hadoop.log.dir/,AVAILABLE} 2024-12-01T14:29:37,304 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T14:29:37,408 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/java.io.tmpdir/jetty-localhost-36309-hadoop-hdfs-3_4_1-tests_jar-_-any-10956179413782261999/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T14:29:37,409 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:36309} 2024-12-01T14:29:37,409 INFO [Time-limited test {}] server.Server(415): Started @3633ms 2024-12-01T14:29:37,412 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T14:29:37,448 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T14:29:37,452 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T14:29:37,455 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T14:29:37,455 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T14:29:37,455 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T14:29:37,456 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/hadoop.log.dir/,AVAILABLE} 2024-12-01T14:29:37,456 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T14:29:37,552 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/java.io.tmpdir/jetty-localhost-41803-hadoop-hdfs-3_4_1-tests_jar-_-any-6506822170724177602/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T14:29:37,553 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:41803} 2024-12-01T14:29:37,553 INFO [Time-limited test {}] server.Server(415): Started @3777ms 2024-12-01T14:29:37,555 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T14:29:39,423 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data4/current/BP-1542046065-172.17.0.3-1733063375577/current, will proceed with Du for space computation calculation, 2024-12-01T14:29:39,423 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data5/current/BP-1542046065-172.17.0.3-1733063375577/current, will proceed with Du for space computation calculation, 2024-12-01T14:29:39,423 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data1/current/BP-1542046065-172.17.0.3-1733063375577/current, will proceed with Du for space computation calculation, 2024-12-01T14:29:39,423 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data2/current/BP-1542046065-172.17.0.3-1733063375577/current, will proceed with Du for space computation calculation, 2024-12-01T14:29:39,423 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data6/current/BP-1542046065-172.17.0.3-1733063375577/current, will proceed with Du for space computation calculation, 2024-12-01T14:29:39,423 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data3/current/BP-1542046065-172.17.0.3-1733063375577/current, will proceed with Du for space computation calculation, 2024-12-01T14:29:39,463 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T14:29:39,463 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T14:29:39,464 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T14:29:39,510 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf624bfa92f4d46d2 with lease ID 0x1412499f7359246e: Processing first storage report for DS-839bb17d-2244-4d81-b1ed-c0592745fdb4 from datanode DatanodeRegistration(127.0.0.1:33423, datanodeUuid=83924cbb-93f5-4467-bf71-2c95bfcc03dc, infoPort=36303, infoSecurePort=0, ipcPort=44683, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577) 2024-12-01T14:29:39,511 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf624bfa92f4d46d2 with lease ID 0x1412499f7359246e: from storage DS-839bb17d-2244-4d81-b1ed-c0592745fdb4 node DatanodeRegistration(127.0.0.1:33423, datanodeUuid=83924cbb-93f5-4467-bf71-2c95bfcc03dc, infoPort=36303, infoSecurePort=0, ipcPort=44683, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-01T14:29:39,512 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2405c9bec5b9095 with lease ID 0x1412499f7359246f: Processing first storage report for DS-0a95010c-7d31-41b3-acf3-b49ac3735c69 from datanode DatanodeRegistration(127.0.0.1:33303, datanodeUuid=b8de6bcc-2406-43e2-9e37-8edacfbb4cc0, infoPort=40289, infoSecurePort=0, ipcPort=36509, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577) 2024-12-01T14:29:39,512 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2405c9bec5b9095 with lease ID 0x1412499f7359246f: from storage DS-0a95010c-7d31-41b3-acf3-b49ac3735c69 node DatanodeRegistration(127.0.0.1:33303, datanodeUuid=b8de6bcc-2406-43e2-9e37-8edacfbb4cc0, infoPort=40289, infoSecurePort=0, ipcPort=36509, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T14:29:39,512 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf44f54a63e06ad00 with lease ID 0x1412499f73592470: Processing first storage report for DS-047d8189-f10b-4007-89ba-d48b429cb5d6 from datanode DatanodeRegistration(127.0.0.1:40063, datanodeUuid=07777e45-b85e-40f3-b473-ca12a96b8269, infoPort=33969, infoSecurePort=0, ipcPort=42983, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577) 2024-12-01T14:29:39,512 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf44f54a63e06ad00 with lease ID 0x1412499f73592470: from storage DS-047d8189-f10b-4007-89ba-d48b429cb5d6 node DatanodeRegistration(127.0.0.1:40063, datanodeUuid=07777e45-b85e-40f3-b473-ca12a96b8269, infoPort=33969, infoSecurePort=0, ipcPort=42983, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T14:29:39,512 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf624bfa92f4d46d2 with lease ID 0x1412499f7359246e: Processing first storage report for DS-81e407cd-3b25-491b-9b22-642d2a34893a from datanode DatanodeRegistration(127.0.0.1:33423, datanodeUuid=83924cbb-93f5-4467-bf71-2c95bfcc03dc, infoPort=36303, infoSecurePort=0, ipcPort=44683, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577) 2024-12-01T14:29:39,513 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf624bfa92f4d46d2 with lease ID 0x1412499f7359246e: from storage DS-81e407cd-3b25-491b-9b22-642d2a34893a node DatanodeRegistration(127.0.0.1:33423, datanodeUuid=83924cbb-93f5-4467-bf71-2c95bfcc03dc, infoPort=36303, infoSecurePort=0, ipcPort=44683, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T14:29:39,513 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2405c9bec5b9095 with lease ID 0x1412499f7359246f: Processing first storage report for DS-a2f1bb62-1e9a-44e2-937a-d9829703f931 from datanode DatanodeRegistration(127.0.0.1:33303, datanodeUuid=b8de6bcc-2406-43e2-9e37-8edacfbb4cc0, infoPort=40289, infoSecurePort=0, ipcPort=36509, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577) 2024-12-01T14:29:39,513 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2405c9bec5b9095 with lease ID 0x1412499f7359246f: from storage DS-a2f1bb62-1e9a-44e2-937a-d9829703f931 node DatanodeRegistration(127.0.0.1:33303, datanodeUuid=b8de6bcc-2406-43e2-9e37-8edacfbb4cc0, infoPort=40289, infoSecurePort=0, ipcPort=36509, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T14:29:39,513 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf44f54a63e06ad00 with lease ID 0x1412499f73592470: Processing first storage report for DS-8f306efc-26d6-4b19-b57b-d285f4583f0c from datanode DatanodeRegistration(127.0.0.1:40063, datanodeUuid=07777e45-b85e-40f3-b473-ca12a96b8269, infoPort=33969, infoSecurePort=0, ipcPort=42983, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577) 2024-12-01T14:29:39,513 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf44f54a63e06ad00 with lease ID 0x1412499f73592470: from storage DS-8f306efc-26d6-4b19-b57b-d285f4583f0c node DatanodeRegistration(127.0.0.1:40063, datanodeUuid=07777e45-b85e-40f3-b473-ca12a96b8269, infoPort=33969, infoSecurePort=0, ipcPort=42983, storageInfo=lv=-57;cid=testClusterID;nsid=195786470;c=1733063375577), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T14:29:39,586 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726 2024-12-01T14:29:39,647 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/zookeeper_0, clientPort=56293, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-01T14:29:39,667 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56293 2024-12-01T14:29:39,675 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:39,680 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:39,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741825_1001 (size=7) 2024-12-01T14:29:39,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741825_1001 (size=7) 2024-12-01T14:29:39,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741825_1001 (size=7) 2024-12-01T14:29:40,265 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9 with version=8 2024-12-01T14:29:40,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/hbase-staging 2024-12-01T14:29:40,543 INFO [Time-limited test {}] client.ConnectionUtils(128): master/57b748500ef9:0 server-side Connection retries=45 2024-12-01T14:29:40,551 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:40,551 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:40,555 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:29:40,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:40,556 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:29:40,675 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-01T14:29:40,734 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-01T14:29:40,744 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-01T14:29:40,747 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:29:40,772 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 81311 (auto-detected) 2024-12-01T14:29:40,773 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-01T14:29:40,791 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34327 2024-12-01T14:29:40,810 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34327 connecting to ZooKeeper ensemble=127.0.0.1:56293 2024-12-01T14:29:40,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:343270x0, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:29:40,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34327-0x101914c865a0000 connected 2024-12-01T14:29:41,067 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:41,074 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:41,086 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:29:41,090 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9, hbase.cluster.distributed=false 2024-12-01T14:29:41,111 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:29:41,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34327 2024-12-01T14:29:41,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34327 2024-12-01T14:29:41,116 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34327 2024-12-01T14:29:41,117 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34327 2024-12-01T14:29:41,117 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34327 2024-12-01T14:29:41,204 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/57b748500ef9:0 server-side Connection retries=45 2024-12-01T14:29:41,205 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:41,205 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:41,205 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:29:41,206 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:41,206 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:29:41,208 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T14:29:41,210 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:29:41,211 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36193 2024-12-01T14:29:41,212 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36193 connecting to ZooKeeper ensemble=127.0.0.1:56293 2024-12-01T14:29:41,213 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:41,216 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:41,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361930x0, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:29:41,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36193-0x101914c865a0001 connected 2024-12-01T14:29:41,229 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:29:41,233 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T14:29:41,241 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T14:29:41,244 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T14:29:41,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:29:41,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36193 2024-12-01T14:29:41,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36193 2024-12-01T14:29:41,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36193 2024-12-01T14:29:41,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36193 2024-12-01T14:29:41,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36193 2024-12-01T14:29:41,265 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/57b748500ef9:0 server-side Connection retries=45 2024-12-01T14:29:41,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:41,265 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:41,266 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:29:41,266 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:41,266 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:29:41,266 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T14:29:41,266 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:29:41,267 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38129 2024-12-01T14:29:41,269 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38129 connecting to ZooKeeper ensemble=127.0.0.1:56293 2024-12-01T14:29:41,271 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:41,274 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:41,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381290x0, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:29:41,286 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:381290x0, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:29:41,286 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38129-0x101914c865a0002 connected 2024-12-01T14:29:41,287 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T14:29:41,287 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T14:29:41,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T14:29:41,290 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:29:41,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38129 2024-12-01T14:29:41,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38129 2024-12-01T14:29:41,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38129 2024-12-01T14:29:41,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38129 2024-12-01T14:29:41,293 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38129 2024-12-01T14:29:41,308 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/57b748500ef9:0 server-side Connection retries=45 2024-12-01T14:29:41,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:41,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:41,308 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T14:29:41,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T14:29:41,309 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T14:29:41,309 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T14:29:41,309 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T14:29:41,310 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40449 2024-12-01T14:29:41,311 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40449 connecting to ZooKeeper ensemble=127.0.0.1:56293 2024-12-01T14:29:41,312 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:41,314 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:41,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404490x0, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T14:29:41,328 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:404490x0, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:29:41,328 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40449-0x101914c865a0003 connected 2024-12-01T14:29:41,328 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T14:29:41,329 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T14:29:41,330 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T14:29:41,332 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T14:29:41,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40449 2024-12-01T14:29:41,333 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40449 2024-12-01T14:29:41,334 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40449 2024-12-01T14:29:41,334 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40449 2024-12-01T14:29:41,335 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40449 2024-12-01T14:29:41,354 DEBUG [M:0;57b748500ef9:34327 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;57b748500ef9:34327 2024-12-01T14:29:41,355 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/57b748500ef9,34327,1733063380396 2024-12-01T14:29:41,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:29:41,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:29:41,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:29:41,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:29:41,372 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/57b748500ef9,34327,1733063380396 2024-12-01T14:29:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T14:29:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T14:29:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T14:29:41,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:41,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:41,409 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-01T14:29:41,411 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/57b748500ef9,34327,1733063380396 from backup master directory 2024-12-01T14:29:41,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/57b748500ef9,34327,1733063380396 2024-12-01T14:29:41,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:29:41,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:29:41,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:29:41,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T14:29:41,423 WARN [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:29:41,424 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=57b748500ef9,34327,1733063380396 2024-12-01T14:29:41,427 INFO [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-01T14:29:41,428 INFO [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-01T14:29:41,479 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/hbase.id] with ID: 13e07f3e-13f5-4a4b-89f8-14b649bbc29b 2024-12-01T14:29:41,479 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/.tmp/hbase.id 2024-12-01T14:29:41,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741826_1002 (size=42) 2024-12-01T14:29:41,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741826_1002 (size=42) 2024-12-01T14:29:41,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741826_1002 (size=42) 2024-12-01T14:29:41,495 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/.tmp/hbase.id]:[hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/hbase.id] 2024-12-01T14:29:41,539 INFO [master/57b748500ef9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T14:29:41,543 INFO [master/57b748500ef9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-01T14:29:41,559 INFO [master/57b748500ef9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-01T14:29:41,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:41,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:41,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:41,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:41,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741827_1003 (size=196) 2024-12-01T14:29:41,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741827_1003 (size=196) 2024-12-01T14:29:41,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741827_1003 (size=196) 2024-12-01T14:29:41,605 INFO [master/57b748500ef9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T14:29:41,607 INFO [master/57b748500ef9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-01T14:29:41,612 INFO [master/57b748500ef9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:41,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741828_1004 (size=1189) 2024-12-01T14:29:41,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741828_1004 (size=1189) 2024-12-01T14:29:41,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741828_1004 (size=1189) 2024-12-01T14:29:41,659 INFO [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store 2024-12-01T14:29:41,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741829_1005 (size=34) 2024-12-01T14:29:41,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741829_1005 (size=34) 2024-12-01T14:29:41,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741829_1005 (size=34) 2024-12-01T14:29:41,687 INFO [master/57b748500ef9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-01T14:29:41,691 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:41,693 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T14:29:41,693 INFO [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:29:41,693 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:29:41,695 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T14:29:41,695 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:29:41,695 INFO [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:29:41,697 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733063381692Disabling compacts and flushes for region at 1733063381692Disabling writes for close at 1733063381695 (+3 ms)Writing region close event to WAL at 1733063381695Closed at 1733063381695 2024-12-01T14:29:41,699 WARN [master/57b748500ef9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/.initializing 2024-12-01T14:29:41,699 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/WALs/57b748500ef9,34327,1733063380396 2024-12-01T14:29:41,706 INFO [master/57b748500ef9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:41,720 INFO [master/57b748500ef9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=57b748500ef9%2C34327%2C1733063380396, suffix=, logDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/WALs/57b748500ef9,34327,1733063380396, archiveDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/oldWALs, maxLogs=10 2024-12-01T14:29:41,747 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/WALs/57b748500ef9,34327,1733063380396/57b748500ef9%2C34327%2C1733063380396.1733063381724, exclude list is [], retry=0 2024-12-01T14:29:41,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:29:41,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:41,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:41,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:41,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-01T14:29:41,805 INFO [master/57b748500ef9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/WALs/57b748500ef9,34327,1733063380396/57b748500ef9%2C34327%2C1733063380396.1733063381724 2024-12-01T14:29:41,806 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:29:41,807 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:41,807 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:41,810 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,811 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,845 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,866 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-01T14:29:41,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:41,873 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:29:41,873 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,877 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-01T14:29:41,877 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:41,878 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:41,878 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-01T14:29:41,881 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:41,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:41,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-01T14:29:41,885 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:41,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:41,887 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,890 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,891 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,895 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,896 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,899 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T14:29:41,903 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T14:29:41,908 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:41,909 INFO [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70681857, jitterRate=0.05324174463748932}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T14:29:41,915 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733063381822Initializing all the Stores at 1733063381824 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733063381825 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063381825Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063381825Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063381825Cleaning up temporary data from old regions at 1733063381896 (+71 ms)Region opened successfully at 1733063381915 (+19 ms) 2024-12-01T14:29:41,916 INFO [master/57b748500ef9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-01T14:29:41,947 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dae27ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=57b748500ef9/172.17.0.3:0 2024-12-01T14:29:41,974 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-01T14:29:41,983 INFO [master/57b748500ef9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-01T14:29:41,983 INFO [master/57b748500ef9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-01T14:29:41,985 INFO [master/57b748500ef9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-01T14:29:41,987 INFO [master/57b748500ef9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-01T14:29:41,991 INFO [master/57b748500ef9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-01T14:29:41,991 INFO [master/57b748500ef9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-01T14:29:42,013 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-01T14:29:42,021 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-01T14:29:42,070 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-01T14:29:42,074 INFO [master/57b748500ef9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-01T14:29:42,077 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-01T14:29:42,085 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-01T14:29:42,087 INFO [master/57b748500ef9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-01T14:29:42,090 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-01T14:29:42,102 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-01T14:29:42,104 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-01T14:29:42,116 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-01T14:29:42,135 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-01T14:29:42,144 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-01T14:29:42,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T14:29:42,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T14:29:42,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T14:29:42,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T14:29:42,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,159 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=57b748500ef9,34327,1733063380396, sessionid=0x101914c865a0000, setting cluster-up flag (Was=false) 2024-12-01T14:29:42,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,222 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-01T14:29:42,226 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=57b748500ef9,34327,1733063380396 2024-12-01T14:29:42,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:42,285 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-01T14:29:42,289 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=57b748500ef9,34327,1733063380396 2024-12-01T14:29:42,299 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-01T14:29:42,338 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(746): ClusterId : 13e07f3e-13f5-4a4b-89f8-14b649bbc29b 2024-12-01T14:29:42,338 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(746): ClusterId : 13e07f3e-13f5-4a4b-89f8-14b649bbc29b 2024-12-01T14:29:42,338 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(746): ClusterId : 13e07f3e-13f5-4a4b-89f8-14b649bbc29b 2024-12-01T14:29:42,341 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T14:29:42,341 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T14:29:42,341 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T14:29:42,357 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T14:29:42,357 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T14:29:42,357 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T14:29:42,357 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T14:29:42,357 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T14:29:42,357 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T14:29:42,368 INFO [AsyncFSWAL-0-hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData-prefix:57b748500ef9,34327,1733063380396 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-12-01T14:29:42,370 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T14:29:42,370 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T14:29:42,370 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T14:29:42,371 DEBUG [RS:0;57b748500ef9:36193 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42216ade, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=57b748500ef9/172.17.0.3:0 2024-12-01T14:29:42,371 DEBUG [RS:2;57b748500ef9:40449 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50752122, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=57b748500ef9/172.17.0.3:0 2024-12-01T14:29:42,371 DEBUG [RS:1;57b748500ef9:38129 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8015ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=57b748500ef9/172.17.0.3:0 2024-12-01T14:29:42,383 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-01T14:29:42,389 DEBUG [RS:0;57b748500ef9:36193 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;57b748500ef9:36193 2024-12-01T14:29:42,389 DEBUG [RS:2;57b748500ef9:40449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;57b748500ef9:40449 2024-12-01T14:29:42,389 DEBUG [RS:1;57b748500ef9:38129 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;57b748500ef9:38129 2024-12-01T14:29:42,393 INFO [RS:2;57b748500ef9:40449 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T14:29:42,393 INFO [RS:0;57b748500ef9:36193 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T14:29:42,393 INFO [RS:1;57b748500ef9:38129 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T14:29:42,393 INFO [RS:2;57b748500ef9:40449 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T14:29:42,393 INFO [RS:1;57b748500ef9:38129 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T14:29:42,393 INFO [RS:0;57b748500ef9:36193 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T14:29:42,393 DEBUG [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T14:29:42,393 DEBUG [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T14:29:42,393 DEBUG [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T14:29:42,395 INFO [master/57b748500ef9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-01T14:29:42,395 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(2659): reportForDuty to master=57b748500ef9,34327,1733063380396 with port=38129, startcode=1733063381264 2024-12-01T14:29:42,395 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(2659): reportForDuty to master=57b748500ef9,34327,1733063380396 with port=36193, startcode=1733063381174 2024-12-01T14:29:42,395 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(2659): reportForDuty to master=57b748500ef9,34327,1733063380396 with port=40449, startcode=1733063381307 2024-12-01T14:29:42,401 INFO [master/57b748500ef9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-01T14:29:42,406 DEBUG [RS:2;57b748500ef9:40449 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T14:29:42,406 DEBUG [RS:0;57b748500ef9:36193 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T14:29:42,406 DEBUG [RS:1;57b748500ef9:38129 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T14:29:42,406 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 57b748500ef9,34327,1733063380396 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-01T14:29:42,413 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/57b748500ef9:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:29:42,414 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/57b748500ef9:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:29:42,414 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/57b748500ef9:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:29:42,414 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/57b748500ef9:0, corePoolSize=5, maxPoolSize=5 2024-12-01T14:29:42,415 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/57b748500ef9:0, corePoolSize=10, maxPoolSize=10 2024-12-01T14:29:42,415 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,415 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/57b748500ef9:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:29:42,415 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,417 INFO [master/57b748500ef9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733063412417 2024-12-01T14:29:42,419 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-01T14:29:42,420 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-01T14:29:42,424 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-01T14:29:42,424 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-01T14:29:42,424 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-01T14:29:42,424 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-01T14:29:42,429 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,432 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-01T14:29:42,434 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-01T14:29:42,434 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-01T14:29:42,435 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T14:29:42,435 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-01T14:29:42,436 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-01T14:29:42,437 INFO [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-01T14:29:42,438 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/57b748500ef9:0:becomeActiveMaster-HFileCleaner.large.0-1733063382438,5,FailOnTimeoutGroup] 2024-12-01T14:29:42,439 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/57b748500ef9:0:becomeActiveMaster-HFileCleaner.small.0-1733063382438,5,FailOnTimeoutGroup] 2024-12-01T14:29:42,439 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,439 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-01T14:29:42,441 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,441 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,442 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60665, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T14:29:42,442 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35903, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T14:29:42,442 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54189, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T14:29:42,443 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:42,444 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-01T14:29:42,449 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34327 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 57b748500ef9,38129,1733063381264 2024-12-01T14:29:42,451 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34327 {}] master.ServerManager(517): Registering regionserver=57b748500ef9,38129,1733063381264 2024-12-01T14:29:42,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741831_1007 (size=1321) 2024-12-01T14:29:42,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741831_1007 (size=1321) 2024-12-01T14:29:42,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741831_1007 (size=1321) 2024-12-01T14:29:42,493 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-01T14:29:42,494 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9 2024-12-01T14:29:42,500 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34327 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 57b748500ef9,36193,1733063381174 2024-12-01T14:29:42,500 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34327 {}] master.ServerManager(517): Registering regionserver=57b748500ef9,36193,1733063381174 2024-12-01T14:29:42,504 DEBUG [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9 2024-12-01T14:29:42,504 DEBUG [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40889 2024-12-01T14:29:42,504 DEBUG [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T14:29:42,505 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34327 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 57b748500ef9,40449,1733063381307 2024-12-01T14:29:42,505 DEBUG [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9 2024-12-01T14:29:42,505 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34327 {}] master.ServerManager(517): Registering regionserver=57b748500ef9,40449,1733063381307 2024-12-01T14:29:42,505 DEBUG [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40889 2024-12-01T14:29:42,505 DEBUG [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T14:29:42,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741832_1008 (size=32) 2024-12-01T14:29:42,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741832_1008 (size=32) 2024-12-01T14:29:42,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741832_1008 (size=32) 2024-12-01T14:29:42,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:42,512 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T14:29:42,514 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T14:29:42,515 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:42,516 DEBUG [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9 2024-12-01T14:29:42,516 DEBUG [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40889 2024-12-01T14:29:42,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:29:42,516 DEBUG [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T14:29:42,516 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T14:29:42,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:29:42,519 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T14:29:42,519 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:42,520 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:29:42,520 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T14:29:42,523 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T14:29:42,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:42,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:29:42,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T14:29:42,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T14:29:42,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:42,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:29:42,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T14:29:42,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740 2024-12-01T14:29:42,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740 2024-12-01T14:29:42,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T14:29:42,533 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T14:29:42,533 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T14:29:42,536 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T14:29:42,539 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:42,540 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65208309, jitterRate=-0.028320476412773132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T14:29:42,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733063382510Initializing all the Stores at 1733063382512 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733063382512Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733063382512Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063382512Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733063382512Cleaning up temporary data from old regions at 1733063382533 (+21 ms)Region opened successfully at 1733063382542 (+9 ms) 2024-12-01T14:29:42,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T14:29:42,543 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T14:29:42,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T14:29:42,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T14:29:42,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T14:29:42,544 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T14:29:42,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733063382543Disabling compacts and flushes for region at 1733063382543Disabling writes for close at 1733063382543Writing region close event to WAL at 1733063382544 (+1 ms)Closed at 1733063382544 2024-12-01T14:29:42,547 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T14:29:42,547 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-01T14:29:42,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T14:29:42,561 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T14:29:42,564 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-01T14:29:42,569 DEBUG [RS:1;57b748500ef9:38129 {}] zookeeper.ZKUtil(111): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/57b748500ef9,38129,1733063381264 2024-12-01T14:29:42,569 DEBUG [RS:0;57b748500ef9:36193 {}] zookeeper.ZKUtil(111): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/57b748500ef9,36193,1733063381174 2024-12-01T14:29:42,570 WARN [RS:0;57b748500ef9:36193 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:29:42,570 WARN [RS:1;57b748500ef9:38129 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:29:42,570 INFO [RS:0;57b748500ef9:36193 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:42,570 INFO [RS:1;57b748500ef9:38129 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:42,570 DEBUG [RS:2;57b748500ef9:40449 {}] zookeeper.ZKUtil(111): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/57b748500ef9,40449,1733063381307 2024-12-01T14:29:42,570 WARN [RS:2;57b748500ef9:40449 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T14:29:42,570 DEBUG [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174 2024-12-01T14:29:42,570 DEBUG [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,38129,1733063381264 2024-12-01T14:29:42,570 INFO [RS:2;57b748500ef9:40449 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:42,570 DEBUG [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,40449,1733063381307 2024-12-01T14:29:42,571 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [57b748500ef9,36193,1733063381174] 2024-12-01T14:29:42,571 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [57b748500ef9,38129,1733063381264] 2024-12-01T14:29:42,572 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [57b748500ef9,40449,1733063381307] 2024-12-01T14:29:42,595 INFO [RS:2;57b748500ef9:40449 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T14:29:42,595 INFO [RS:0;57b748500ef9:36193 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T14:29:42,595 INFO [RS:1;57b748500ef9:38129 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T14:29:42,608 INFO [RS:1;57b748500ef9:38129 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T14:29:42,608 INFO [RS:0;57b748500ef9:36193 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T14:29:42,608 INFO [RS:2;57b748500ef9:40449 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T14:29:42,614 INFO [RS:1;57b748500ef9:38129 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T14:29:42,614 INFO [RS:2;57b748500ef9:40449 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T14:29:42,614 INFO [RS:0;57b748500ef9:36193 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T14:29:42,614 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,614 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,614 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,615 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T14:29:42,616 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T14:29:42,620 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T14:29:42,621 INFO [RS:0;57b748500ef9:36193 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T14:29:42,621 INFO [RS:1;57b748500ef9:38129 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T14:29:42,621 INFO [RS:2;57b748500ef9:40449 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T14:29:42,623 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,623 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,623 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,623 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,623 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,623 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,623 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,623 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,623 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,623 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,623 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:29:42,624 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:29:42,624 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0, corePoolSize=2, maxPoolSize=2 2024-12-01T14:29:42,624 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,624 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,625 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,625 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/57b748500ef9:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:29:42,625 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/57b748500ef9:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:29:42,625 DEBUG [RS:1;57b748500ef9:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/57b748500ef9:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:29:42,625 DEBUG [RS:0;57b748500ef9:36193 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/57b748500ef9:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:29:42,625 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,625 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,625 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/57b748500ef9:0, corePoolSize=1, maxPoolSize=1 2024-12-01T14:29:42,625 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/57b748500ef9:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:29:42,625 DEBUG [RS:2;57b748500ef9:40449 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/57b748500ef9:0, corePoolSize=3, maxPoolSize=3 2024-12-01T14:29:42,627 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,40449,1733063381307-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,36193,1733063381174-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,627 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,38129,1733063381264-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:29:42,645 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T14:29:42,645 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T14:29:42,645 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T14:29:42,647 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,36193,1733063381174-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,647 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,38129,1733063381264-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,647 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,40449,1733063381307-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,647 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,647 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,647 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,647 INFO [RS:1;57b748500ef9:38129 {}] regionserver.Replication(171): 57b748500ef9,38129,1733063381264 started 2024-12-01T14:29:42,647 INFO [RS:2;57b748500ef9:40449 {}] regionserver.Replication(171): 57b748500ef9,40449,1733063381307 started 2024-12-01T14:29:42,647 INFO [RS:0;57b748500ef9:36193 {}] regionserver.Replication(171): 57b748500ef9,36193,1733063381174 started 2024-12-01T14:29:42,665 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,665 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,666 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(1482): Serving as 57b748500ef9,36193,1733063381174, RpcServer on 57b748500ef9/172.17.0.3:36193, sessionid=0x101914c865a0001 2024-12-01T14:29:42,666 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(1482): Serving as 57b748500ef9,38129,1733063381264, RpcServer on 57b748500ef9/172.17.0.3:38129, sessionid=0x101914c865a0002 2024-12-01T14:29:42,666 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T14:29:42,666 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T14:29:42,666 DEBUG [RS:0;57b748500ef9:36193 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 57b748500ef9,36193,1733063381174 2024-12-01T14:29:42,666 DEBUG [RS:1;57b748500ef9:38129 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 57b748500ef9,38129,1733063381264 2024-12-01T14:29:42,667 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '57b748500ef9,36193,1733063381174' 2024-12-01T14:29:42,667 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '57b748500ef9,38129,1733063381264' 2024-12-01T14:29:42,667 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T14:29:42,667 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T14:29:42,668 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T14:29:42,668 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T14:29:42,668 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T14:29:42,668 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T14:29:42,669 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T14:29:42,669 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T14:29:42,669 DEBUG [RS:1;57b748500ef9:38129 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 57b748500ef9,38129,1733063381264 2024-12-01T14:29:42,669 DEBUG [RS:0;57b748500ef9:36193 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 57b748500ef9,36193,1733063381174 2024-12-01T14:29:42,669 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '57b748500ef9,38129,1733063381264' 2024-12-01T14:29:42,669 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '57b748500ef9,36193,1733063381174' 2024-12-01T14:29:42,669 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T14:29:42,669 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T14:29:42,669 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T14:29:42,669 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T14:29:42,670 DEBUG [RS:1;57b748500ef9:38129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T14:29:42,670 DEBUG [RS:0;57b748500ef9:36193 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T14:29:42,670 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:42,670 INFO [RS:0;57b748500ef9:36193 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T14:29:42,670 INFO [RS:1;57b748500ef9:38129 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T14:29:42,670 INFO [RS:0;57b748500ef9:36193 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T14:29:42,670 INFO [RS:1;57b748500ef9:38129 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T14:29:42,670 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(1482): Serving as 57b748500ef9,40449,1733063381307, RpcServer on 57b748500ef9/172.17.0.3:40449, sessionid=0x101914c865a0003 2024-12-01T14:29:42,670 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T14:29:42,670 DEBUG [RS:2;57b748500ef9:40449 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 57b748500ef9,40449,1733063381307 2024-12-01T14:29:42,671 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '57b748500ef9,40449,1733063381307' 2024-12-01T14:29:42,671 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T14:29:42,671 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T14:29:42,672 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T14:29:42,672 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T14:29:42,672 DEBUG [RS:2;57b748500ef9:40449 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 57b748500ef9,40449,1733063381307 2024-12-01T14:29:42,672 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '57b748500ef9,40449,1733063381307' 2024-12-01T14:29:42,672 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T14:29:42,673 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T14:29:42,673 DEBUG [RS:2;57b748500ef9:40449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T14:29:42,673 INFO [RS:2;57b748500ef9:40449 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T14:29:42,673 INFO [RS:2;57b748500ef9:40449 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T14:29:42,715 WARN [57b748500ef9:34327 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T14:29:42,774 INFO [RS:2;57b748500ef9:40449 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:42,774 INFO [RS:0;57b748500ef9:36193 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:42,774 INFO [RS:1;57b748500ef9:38129 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:42,777 INFO [RS:0;57b748500ef9:36193 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=57b748500ef9%2C36193%2C1733063381174, suffix=, logDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174, archiveDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/oldWALs, maxLogs=32 2024-12-01T14:29:42,777 INFO [RS:1;57b748500ef9:38129 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=57b748500ef9%2C38129%2C1733063381264, suffix=, logDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,38129,1733063381264, archiveDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/oldWALs, maxLogs=32 2024-12-01T14:29:42,777 INFO [RS:2;57b748500ef9:40449 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=57b748500ef9%2C40449%2C1733063381307, suffix=, logDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,40449,1733063381307, archiveDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/oldWALs, maxLogs=32 2024-12-01T14:29:42,794 DEBUG [RS:2;57b748500ef9:40449 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,40449,1733063381307/57b748500ef9%2C40449%2C1733063381307.1733063382781, exclude list is [], retry=0 2024-12-01T14:29:42,794 DEBUG [RS:1;57b748500ef9:38129 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,38129,1733063381264/57b748500ef9%2C38129%2C1733063381264.1733063382781, exclude list is [], retry=0 2024-12-01T14:29:42,794 DEBUG [RS:0;57b748500ef9:36193 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174/57b748500ef9%2C36193%2C1733063381174.1733063382781, exclude list is [], retry=0 2024-12-01T14:29:42,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:42,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:42,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:42,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:42,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:42,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:42,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:42,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:42,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:42,806 INFO [RS:1;57b748500ef9:38129 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,38129,1733063381264/57b748500ef9%2C38129%2C1733063381264.1733063382781 2024-12-01T14:29:42,806 INFO [RS:0;57b748500ef9:36193 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174/57b748500ef9%2C36193%2C1733063381174.1733063382781 2024-12-01T14:29:42,806 INFO [RS:2;57b748500ef9:40449 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,40449,1733063381307/57b748500ef9%2C40449%2C1733063381307.1733063382781 2024-12-01T14:29:42,807 DEBUG [RS:1;57b748500ef9:38129 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:29:42,807 DEBUG [RS:0;57b748500ef9:36193 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:29:42,807 DEBUG [RS:2;57b748500ef9:40449 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:29:42,970 DEBUG [57b748500ef9:34327 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-01T14:29:42,980 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(204): Hosts are {57b748500ef9=0} racks are {/default-rack=0} 2024-12-01T14:29:42,985 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T14:29:42,985 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T14:29:42,985 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-01T14:29:42,986 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T14:29:42,986 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T14:29:42,986 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-01T14:29:42,986 INFO [57b748500ef9:34327 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T14:29:42,986 INFO [57b748500ef9:34327 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T14:29:42,986 INFO [57b748500ef9:34327 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-01T14:29:42,986 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T14:29:42,992 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=57b748500ef9,38129,1733063381264 2024-12-01T14:29:42,997 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 57b748500ef9,38129,1733063381264, state=OPENING 2024-12-01T14:29:43,049 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-01T14:29:43,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:43,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:43,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:43,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:29:43,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:29:43,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:29:43,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:29:43,063 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:29:43,066 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T14:29:43,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=57b748500ef9,38129,1733063381264}] 2024-12-01T14:29:43,248 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T14:29:43,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38241, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T14:29:43,262 INFO [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T14:29:43,262 INFO [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:43,263 INFO [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T14:29:43,266 INFO [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=57b748500ef9%2C38129%2C1733063381264.meta, suffix=.meta, logDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,38129,1733063381264, archiveDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/oldWALs, maxLogs=32 2024-12-01T14:29:43,280 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,38129,1733063381264/57b748500ef9%2C38129%2C1733063381264.meta.1733063383267.meta, exclude list is [], retry=0 2024-12-01T14:29:43,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:43,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:43,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:43,287 INFO [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,38129,1733063381264/57b748500ef9%2C38129%2C1733063381264.meta.1733063383267.meta 2024-12-01T14:29:43,287 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:29:43,287 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:43,289 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T14:29:43,291 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T14:29:43,295 INFO [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T14:29:43,299 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T14:29:43,300 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:43,300 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T14:29:43,300 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T14:29:43,303 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T14:29:43,304 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T14:29:43,305 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:43,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:29:43,306 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T14:29:43,307 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T14:29:43,307 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:43,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:29:43,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T14:29:43,310 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T14:29:43,310 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:43,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:29:43,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T14:29:43,312 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T14:29:43,312 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:43,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T14:29:43,313 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T14:29:43,314 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740 2024-12-01T14:29:43,317 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740 2024-12-01T14:29:43,320 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T14:29:43,320 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T14:29:43,321 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T14:29:43,323 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T14:29:43,325 INFO [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64727804, jitterRate=-0.0354805588722229}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T14:29:43,325 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T14:29:43,326 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733063383300Writing region info on filesystem at 1733063383301 (+1 ms)Initializing all the Stores at 1733063383302 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733063383302Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733063383303 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063383303Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733063383303Cleaning up temporary data from old regions at 1733063383320 (+17 ms)Running coprocessor post-open hooks at 1733063383325 (+5 ms)Region opened successfully at 1733063383326 (+1 ms) 2024-12-01T14:29:43,333 INFO [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733063383240 2024-12-01T14:29:43,343 DEBUG [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T14:29:43,343 INFO [RS_OPEN_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T14:29:43,345 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=57b748500ef9,38129,1733063381264 2024-12-01T14:29:43,348 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 57b748500ef9,38129,1733063381264, state=OPEN 2024-12-01T14:29:43,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:29:43,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:29:43,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:29:43,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T14:29:43,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:29:43,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:29:43,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:29:43,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T14:29:43,366 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=57b748500ef9,38129,1733063381264 2024-12-01T14:29:43,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-01T14:29:43,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=57b748500ef9,38129,1733063381264 in 298 msec 2024-12-01T14:29:43,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-01T14:29:43,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 822 msec 2024-12-01T14:29:43,382 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T14:29:43,382 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-01T14:29:43,401 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T14:29:43,402 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=57b748500ef9,38129,1733063381264, seqNum=-1] 2024-12-01T14:29:43,423 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T14:29:43,425 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42577, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T14:29:43,444 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1160 sec 2024-12-01T14:29:43,444 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733063383444, completionTime=-1 2024-12-01T14:29:43,447 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-01T14:29:43,448 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-01T14:29:43,495 INFO [master/57b748500ef9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-01T14:29:43,495 INFO [master/57b748500ef9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733063443495 2024-12-01T14:29:43,495 INFO [master/57b748500ef9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733063503495 2024-12-01T14:29:43,495 INFO [master/57b748500ef9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 47 msec 2024-12-01T14:29:43,496 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-01T14:29:43,506 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,34327,1733063380396-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:43,507 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,34327,1733063380396-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:43,507 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,34327,1733063380396-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:43,508 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-57b748500ef9:34327, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:43,508 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:43,509 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:43,515 DEBUG [master/57b748500ef9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-01T14:29:43,535 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.111sec 2024-12-01T14:29:43,537 INFO [master/57b748500ef9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-01T14:29:43,538 INFO [master/57b748500ef9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-01T14:29:43,539 INFO [master/57b748500ef9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-01T14:29:43,539 INFO [master/57b748500ef9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-01T14:29:43,539 INFO [master/57b748500ef9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-01T14:29:43,540 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,34327,1733063380396-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T14:29:43,541 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,34327,1733063380396-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-01T14:29:43,546 DEBUG [master/57b748500ef9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-01T14:29:43,547 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-01T14:29:43,548 INFO [master/57b748500ef9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=57b748500ef9,34327,1733063380396-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:43,549 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6e3190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T14:29:43,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 57b748500ef9,34327,-1 for getting cluster id 2024-12-01T14:29:43,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T14:29:43,561 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13e07f3e-13f5-4a4b-89f8-14b649bbc29b' 2024-12-01T14:29:43,563 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T14:29:43,563 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13e07f3e-13f5-4a4b-89f8-14b649bbc29b" 2024-12-01T14:29:43,564 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1027195a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T14:29:43,564 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [57b748500ef9,34327,-1] 2024-12-01T14:29:43,567 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T14:29:43,568 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:29:43,570 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T14:29:43,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f808dbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T14:29:43,573 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T14:29:43,579 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=57b748500ef9,38129,1733063381264, seqNum=-1] 2024-12-01T14:29:43,580 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T14:29:43,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T14:29:43,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=57b748500ef9,34327,1733063380396 2024-12-01T14:29:43,601 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:40889/hbase 2024-12-01T14:29:43,614 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=368, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=219, ProcessCount=11, AvailableMemoryMB=5347 2024-12-01T14:29:43,631 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:43,634 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:43,635 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:43,639 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-32563927, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/hregion-32563927, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:43,654 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-32563927/hregion-32563927.1733063383640, exclude list is [], retry=0 2024-12-01T14:29:43,659 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:43,660 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:43,660 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:43,665 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-32563927/hregion-32563927.1733063383640 2024-12-01T14:29:43,667 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:29:43,668 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f7f97033b86e1a61da15ec7d474ebffd, NAME => 'testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/hbase 2024-12-01T14:29:43,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741838_1014 (size=64) 2024-12-01T14:29:43,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741838_1014 (size=64) 2024-12-01T14:29:43,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741838_1014 (size=64) 2024-12-01T14:29:43,684 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:43,688 INFO [StoreOpener-f7f97033b86e1a61da15ec7d474ebffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,691 INFO [StoreOpener-f7f97033b86e1a61da15ec7d474ebffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7f97033b86e1a61da15ec7d474ebffd columnFamilyName a 2024-12-01T14:29:43,691 DEBUG [StoreOpener-f7f97033b86e1a61da15ec7d474ebffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:43,692 INFO [StoreOpener-f7f97033b86e1a61da15ec7d474ebffd-1 {}] regionserver.HStore(327): Store=f7f97033b86e1a61da15ec7d474ebffd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:43,692 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,694 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,695 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,695 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,697 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,701 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:43,702 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f7f97033b86e1a61da15ec7d474ebffd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59655050, jitterRate=-0.11107048392295837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:29:43,703 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f7f97033b86e1a61da15ec7d474ebffd: Writing region info on filesystem at 1733063383684Initializing all the Stores at 1733063383686 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063383686Cleaning up temporary data from old regions at 1733063383695 (+9 ms)Region opened successfully at 1733063383703 (+8 ms) 2024-12-01T14:29:43,703 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f7f97033b86e1a61da15ec7d474ebffd, disabling compactions & flushes 2024-12-01T14:29:43,703 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd. 2024-12-01T14:29:43,703 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd. 2024-12-01T14:29:43,703 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd. after waiting 0 ms 2024-12-01T14:29:43,703 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd. 2024-12-01T14:29:43,704 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd. 2024-12-01T14:29:43,704 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f7f97033b86e1a61da15ec7d474ebffd: Waiting for close lock at 1733063383703Disabling compacts and flushes for region at 1733063383703Disabling writes for close at 1733063383703Writing region close event to WAL at 1733063383704 (+1 ms)Closed at 1733063383704 2024-12-01T14:29:43,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741837_1013 (size=95) 2024-12-01T14:29:43,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741837_1013 (size=95) 2024-12-01T14:29:43,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741837_1013 (size=95) 2024-12-01T14:29:43,719 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:29:43,719 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-32563927:(num 1733063383640) 2024-12-01T14:29:43,722 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T14:29:43,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741839_1015 (size=320) 2024-12-01T14:29:43,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741839_1015 (size=320) 2024-12-01T14:29:43,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741839_1015 (size=320) 2024-12-01T14:29:43,741 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T14:29:43,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741840_1016 (size=253) 2024-12-01T14:29:43,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741840_1016 (size=253) 2024-12-01T14:29:43,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741840_1016 (size=253) 2024-12-01T14:29:43,775 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1, size=320 (320bytes) 2024-12-01T14:29:43,775 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-01T14:29:43,776 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-01T14:29:43,776 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1 2024-12-01T14:29:43,781 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1 after 3ms 2024-12-01T14:29:43,787 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:43,788 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1 took 14ms 2024-12-01T14:29:43,798 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1 so closing down 2024-12-01T14:29:43,798 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:43,801 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-01T14:29:43,803 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000001-wal-1.temp 2024-12-01T14:29:43,804 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:43,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741841_1017 (size=320) 2024-12-01T14:29:43,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741841_1017 (size=320) 2024-12-01T14:29:43,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741841_1017 (size=320) 2024-12-01T14:29:43,817 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-01T14:29:43,820 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002 2024-12-01T14:29:43,824 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-12-01T14:29:43,824 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1, journal: Splitting hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1, size=320 (320bytes) at 1733063383775Finishing writing output for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1 so closing down at 1733063383798 (+23 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000001-wal-1.temp at 1733063383803 (+5 ms)3 split writer threads finished at 1733063383804 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733063383817 (+13 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002 at 1733063383820 (+3 ms)Processed 2 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1733063383824 (+4 ms) 2024-12-01T14:29:43,838 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2, size=253 (253bytes) 2024-12-01T14:29:43,838 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2 2024-12-01T14:29:43,839 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2 after 1ms 2024-12-01T14:29:43,843 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:43,844 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2 took 5ms 2024-12-01T14:29:43,846 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2 so closing down 2024-12-01T14:29:43,846 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:43,849 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-01T14:29:43,851 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002-wal-2.temp 2024-12-01T14:29:43,851 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:43,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741842_1018 (size=253) 2024-12-01T14:29:43,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741842_1018 (size=253) 2024-12-01T14:29:43,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741842_1018 (size=253) 2024-12-01T14:29:43,862 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-01T14:29:43,866 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:43,868 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-12-01T14:29:43,870 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-12-01T14:29:43,870 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2, journal: Splitting hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2, size=253 (253bytes) at 1733063383838Finishing writing output for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2 so closing down at 1733063383846 (+8 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002-wal-2.temp at 1733063383851 (+5 ms)3 split writer threads finished at 1733063383851Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733063383862 (+11 ms)Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1733063383870 (+8 ms) 2024-12-01T14:29:43,870 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:43,873 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:43,885 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal.1733063383874, exclude list is [], retry=0 2024-12-01T14:29:43,890 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:43,890 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:43,891 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:43,893 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal.1733063383874 2024-12-01T14:29:43,893 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:29:43,894 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f7f97033b86e1a61da15ec7d474ebffd, NAME => 'testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:43,894 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:43,894 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,894 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,896 INFO [StoreOpener-f7f97033b86e1a61da15ec7d474ebffd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,898 INFO [StoreOpener-f7f97033b86e1a61da15ec7d474ebffd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7f97033b86e1a61da15ec7d474ebffd columnFamilyName a 2024-12-01T14:29:43,898 DEBUG [StoreOpener-f7f97033b86e1a61da15ec7d474ebffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:43,899 INFO [StoreOpener-f7f97033b86e1a61da15ec7d474ebffd-1 {}] regionserver.HStore(327): Store=f7f97033b86e1a61da15ec7d474ebffd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:43,900 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,901 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,905 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:43,906 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002 2024-12-01T14:29:43,910 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:43,916 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002 2024-12-01T14:29:43,919 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f7f97033b86e1a61da15ec7d474ebffd 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-01T14:29:43,964 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/.tmp/a/0c88191b6d1b49cbb5dd98e6f9e8789f is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733063383719/Put/seqid=0 2024-12-01T14:29:43,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741844_1020 (size=5170) 2024-12-01T14:29:43,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741844_1020 (size=5170) 2024-12-01T14:29:43,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741844_1020 (size=5170) 2024-12-01T14:29:43,984 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/.tmp/a/0c88191b6d1b49cbb5dd98e6f9e8789f 2024-12-01T14:29:44,027 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/.tmp/a/0c88191b6d1b49cbb5dd98e6f9e8789f as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/a/0c88191b6d1b49cbb5dd98e6f9e8789f 2024-12-01T14:29:44,038 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/a/0c88191b6d1b49cbb5dd98e6f9e8789f, entries=2, sequenceid=2, filesize=5.0 K 2024-12-01T14:29:44,044 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for f7f97033b86e1a61da15ec7d474ebffd in 123ms, sequenceid=2, compaction requested=false; wal=null 2024-12-01T14:29:44,046 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/0000000000000000002 2024-12-01T14:29:44,047 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:44,047 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:44,050 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f7f97033b86e1a61da15ec7d474ebffd 2024-12-01T14:29:44,054 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/f7f97033b86e1a61da15ec7d474ebffd/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-01T14:29:44,055 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f7f97033b86e1a61da15ec7d474ebffd; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70714891, jitterRate=0.05373398959636688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:29:44,056 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f7f97033b86e1a61da15ec7d474ebffd: Writing region info on filesystem at 1733063383894Initializing all the Stores at 1733063383896 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063383896Obtaining lock to block concurrent updates at 1733063383919 (+23 ms)Preparing flush snapshotting stores in f7f97033b86e1a61da15ec7d474ebffd at 1733063383920 (+1 ms)Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733063383922 (+2 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733063383632.f7f97033b86e1a61da15ec7d474ebffd. at 1733063383922Flushing f7f97033b86e1a61da15ec7d474ebffd/a: creating writer at 1733063383924 (+2 ms)Flushing f7f97033b86e1a61da15ec7d474ebffd/a: appending metadata at 1733063383955 (+31 ms)Flushing f7f97033b86e1a61da15ec7d474ebffd/a: closing flushed file at 1733063383957 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5921194b: reopening flushed file at 1733063384026 (+69 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for f7f97033b86e1a61da15ec7d474ebffd in 123ms, sequenceid=2, compaction requested=false; wal=null at 1733063384044 (+18 ms)Cleaning up temporary data from old regions at 1733063384047 (+3 ms)Region opened successfully at 1733063384056 (+9 ms) 2024-12-01T14:29:44,084 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=380 (was 368) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:54370 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:40889/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:52108 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:38316 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:34026 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:51978 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:34008 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=701 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=219 (was 219), ProcessCount=11 (was 11), AvailableMemoryMB=5317 (was 5347) 2024-12-01T14:29:44,094 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=380, OpenFileDescriptor=701, MaxFileDescriptor=1048576, SystemLoadAverage=219, ProcessCount=11, AvailableMemoryMB=5315 2024-12-01T14:29:44,107 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:44,110 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:44,111 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:44,115 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-78163851, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/hregion-78163851, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:44,128 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-78163851/hregion-78163851.1733063384116, exclude list is [], retry=0 2024-12-01T14:29:44,132 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:44,132 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:44,132 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:44,137 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-78163851/hregion-78163851.1733063384116 2024-12-01T14:29:44,137 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:29:44,137 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 11e45ab569b951cdfd8843e115f562b3, NAME => 'testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/hbase 2024-12-01T14:29:44,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741846_1022 (size=64) 2024-12-01T14:29:44,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741846_1022 (size=64) 2024-12-01T14:29:44,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741846_1022 (size=64) 2024-12-01T14:29:44,155 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:44,157 INFO [StoreOpener-11e45ab569b951cdfd8843e115f562b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,159 INFO [StoreOpener-11e45ab569b951cdfd8843e115f562b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e45ab569b951cdfd8843e115f562b3 columnFamilyName a 2024-12-01T14:29:44,159 DEBUG [StoreOpener-11e45ab569b951cdfd8843e115f562b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:44,160 INFO [StoreOpener-11e45ab569b951cdfd8843e115f562b3-1 {}] regionserver.HStore(327): Store=11e45ab569b951cdfd8843e115f562b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:44,160 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,162 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,162 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,163 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,163 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,166 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,169 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:44,171 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 11e45ab569b951cdfd8843e115f562b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74659951, jitterRate=0.11251996457576752}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:29:44,171 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 11e45ab569b951cdfd8843e115f562b3: Writing region info on filesystem at 1733063384155Initializing all the Stores at 1733063384157 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063384157Cleaning up temporary data from old regions at 1733063384163 (+6 ms)Region opened successfully at 1733063384171 (+8 ms) 2024-12-01T14:29:44,171 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 11e45ab569b951cdfd8843e115f562b3, disabling compactions & flushes 2024-12-01T14:29:44,171 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3. 2024-12-01T14:29:44,172 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3. 2024-12-01T14:29:44,172 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3. after waiting 0 ms 2024-12-01T14:29:44,172 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3. 2024-12-01T14:29:44,172 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3. 2024-12-01T14:29:44,172 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 11e45ab569b951cdfd8843e115f562b3: Waiting for close lock at 1733063384171Disabling compacts and flushes for region at 1733063384171Disabling writes for close at 1733063384172 (+1 ms)Writing region close event to WAL at 1733063384172Closed at 1733063384172 2024-12-01T14:29:44,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741845_1021 (size=95) 2024-12-01T14:29:44,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741845_1021 (size=95) 2024-12-01T14:29:44,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741845_1021 (size=95) 2024-12-01T14:29:44,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:29:44,180 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-78163851:(num 1733063384116) 2024-12-01T14:29:44,182 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T14:29:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741847_1023 (size=320) 2024-12-01T14:29:44,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741847_1023 (size=320) 2024-12-01T14:29:44,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741847_1023 (size=320) 2024-12-01T14:29:44,198 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-01T14:29:44,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741848_1024 (size=253) 2024-12-01T14:29:44,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741848_1024 (size=253) 2024-12-01T14:29:44,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741848_1024 (size=253) 2024-12-01T14:29:44,233 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2, size=253 (253bytes) 2024-12-01T14:29:44,233 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2 2024-12-01T14:29:44,234 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2 after 1ms 2024-12-01T14:29:44,238 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:44,238 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2 took 5ms 2024-12-01T14:29:44,241 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2 so closing down 2024-12-01T14:29:44,241 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:44,243 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-01T14:29:44,246 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002-wal-2.temp 2024-12-01T14:29:44,246 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:44,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741849_1025 (size=253) 2024-12-01T14:29:44,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741849_1025 (size=253) 2024-12-01T14:29:44,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741849_1025 (size=253) 2024-12-01T14:29:44,255 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-01T14:29:44,258 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002 2024-12-01T14:29:44,258 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-12-01T14:29:44,259 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2, journal: Splitting hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2, size=253 (253bytes) at 1733063384233Finishing writing output for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2 so closing down at 1733063384241 (+8 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002-wal-2.temp at 1733063384246 (+5 ms)3 split writer threads finished at 1733063384246Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733063384256 (+10 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002 at 1733063384258 (+2 ms)Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1733063384258 2024-12-01T14:29:44,273 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1, size=320 (320bytes) 2024-12-01T14:29:44,274 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1 2024-12-01T14:29:44,274 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1 after 0ms 2024-12-01T14:29:44,278 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:44,278 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1 took 5ms 2024-12-01T14:29:44,281 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1 so closing down 2024-12-01T14:29:44,281 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:44,283 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-01T14:29:44,285 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000001-wal-1.temp 2024-12-01T14:29:44,285 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:44,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741850_1026 (size=320) 2024-12-01T14:29:44,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741850_1026 (size=320) 2024-12-01T14:29:44,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741850_1026 (size=320) 2024-12-01T14:29:44,294 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-01T14:29:44,297 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:44,300 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002, length=253 2024-12-01T14:29:44,303 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002 2024-12-01T14:29:44,303 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-12-01T14:29:44,303 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1, journal: Splitting hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1, size=320 (320bytes) at 1733063384274Finishing writing output for hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1 so closing down at 1733063384281 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000001-wal-1.temp at 1733063384285 (+4 ms)3 split writer threads finished at 1733063384285Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733063384294 (+9 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002 at 1733063384303 (+9 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1733063384303 2024-12-01T14:29:44,303 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:44,306 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:44,318 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal.1733063384306, exclude list is [], retry=0 2024-12-01T14:29:44,321 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:44,322 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:44,323 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:44,324 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal.1733063384306 2024-12-01T14:29:44,325 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:29:44,325 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 11e45ab569b951cdfd8843e115f562b3, NAME => 'testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:44,325 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:44,325 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,325 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,327 INFO [StoreOpener-11e45ab569b951cdfd8843e115f562b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,330 INFO [StoreOpener-11e45ab569b951cdfd8843e115f562b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11e45ab569b951cdfd8843e115f562b3 columnFamilyName a 2024-12-01T14:29:44,330 DEBUG [StoreOpener-11e45ab569b951cdfd8843e115f562b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:44,330 INFO [StoreOpener-11e45ab569b951cdfd8843e115f562b3-1 {}] regionserver.HStore(327): Store=11e45ab569b951cdfd8843e115f562b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:44,331 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,331 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,333 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,334 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002 2024-12-01T14:29:44,337 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:44,339 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002 2024-12-01T14:29:44,339 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 11e45ab569b951cdfd8843e115f562b3 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-01T14:29:44,354 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/.tmp/a/35a23c31e0e24af98ea601b6ba6cd386 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733063384181/Put/seqid=0 2024-12-01T14:29:44,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741852_1028 (size=5170) 2024-12-01T14:29:44,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741852_1028 (size=5170) 2024-12-01T14:29:44,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741852_1028 (size=5170) 2024-12-01T14:29:44,364 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/.tmp/a/35a23c31e0e24af98ea601b6ba6cd386 2024-12-01T14:29:44,373 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/.tmp/a/35a23c31e0e24af98ea601b6ba6cd386 as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/a/35a23c31e0e24af98ea601b6ba6cd386 2024-12-01T14:29:44,382 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/a/35a23c31e0e24af98ea601b6ba6cd386, entries=2, sequenceid=2, filesize=5.0 K 2024-12-01T14:29:44,383 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 11e45ab569b951cdfd8843e115f562b3 in 43ms, sequenceid=2, compaction requested=false; wal=null 2024-12-01T14:29:44,383 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/0000000000000000002 2024-12-01T14:29:44,384 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,384 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,386 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 11e45ab569b951cdfd8843e115f562b3 2024-12-01T14:29:44,389 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/11e45ab569b951cdfd8843e115f562b3/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-01T14:29:44,391 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 11e45ab569b951cdfd8843e115f562b3; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62472944, jitterRate=-0.06908059120178223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:29:44,391 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 11e45ab569b951cdfd8843e115f562b3: Writing region info on filesystem at 1733063384325Initializing all the Stores at 1733063384327 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063384327Obtaining lock to block concurrent updates at 1733063384339 (+12 ms)Preparing flush snapshotting stores in 11e45ab569b951cdfd8843e115f562b3 at 1733063384339Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733063384339Flushing stores of testReplayEditsWrittenIntoWAL,,1733063384108.11e45ab569b951cdfd8843e115f562b3. at 1733063384339Flushing 11e45ab569b951cdfd8843e115f562b3/a: creating writer at 1733063384339Flushing 11e45ab569b951cdfd8843e115f562b3/a: appending metadata at 1733063384353 (+14 ms)Flushing 11e45ab569b951cdfd8843e115f562b3/a: closing flushed file at 1733063384353Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b7c210: reopening flushed file at 1733063384371 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 11e45ab569b951cdfd8843e115f562b3 in 43ms, sequenceid=2, compaction requested=false; wal=null at 1733063384383 (+12 ms)Cleaning up temporary data from old regions at 1733063384384 (+1 ms)Region opened successfully at 1733063384391 (+7 ms) 2024-12-01T14:29:44,407 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=390 (was 380) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:34074 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:38316 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:34026 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:54442 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:52186 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:51978 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 701) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=219 (was 219), ProcessCount=11 (was 11), AvailableMemoryMB=5307 (was 5315) 2024-12-01T14:29:44,417 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=390, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=219, ProcessCount=11, AvailableMemoryMB=5306 2024-12-01T14:29:44,431 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:44,434 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:44,434 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:44,437 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-53100430, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/hregion-53100430, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:44,449 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-53100430/hregion-53100430.1733063384438, exclude list is [], retry=0 2024-12-01T14:29:44,453 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:44,453 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:44,453 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:44,456 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-53100430/hregion-53100430.1733063384438 2024-12-01T14:29:44,456 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:29:44,456 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 881998cd3b67297a80c91841da1e0396, NAME => 'testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/hbase 2024-12-01T14:29:44,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741854_1030 (size=64) 2024-12-01T14:29:44,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741854_1030 (size=64) 2024-12-01T14:29:44,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741854_1030 (size=64) 2024-12-01T14:29:44,467 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:44,469 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:44,471 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 881998cd3b67297a80c91841da1e0396 columnFamilyName a 2024-12-01T14:29:44,471 DEBUG [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:44,472 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(327): Store=881998cd3b67297a80c91841da1e0396/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:44,472 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:44,474 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 881998cd3b67297a80c91841da1e0396 columnFamilyName b 2024-12-01T14:29:44,474 DEBUG [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:44,474 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(327): Store=881998cd3b67297a80c91841da1e0396/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:44,475 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:44,477 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 881998cd3b67297a80c91841da1e0396 columnFamilyName c 2024-12-01T14:29:44,477 DEBUG [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:44,478 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(327): Store=881998cd3b67297a80c91841da1e0396/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:44,478 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:44,480 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:44,480 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:44,482 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:44,482 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:44,483 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:44,484 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:44,487 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:44,487 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 881998cd3b67297a80c91841da1e0396; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70879924, jitterRate=0.05619317293167114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:44,488 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 881998cd3b67297a80c91841da1e0396: Writing region info on filesystem at 1733063384467Initializing all the Stores at 1733063384469 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063384469Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063384469Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063384469Cleaning up temporary data from old regions at 1733063384482 (+13 ms)Region opened successfully at 1733063384487 (+5 ms) 2024-12-01T14:29:44,488 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 881998cd3b67297a80c91841da1e0396, disabling compactions & flushes 2024-12-01T14:29:44,488 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. 2024-12-01T14:29:44,488 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. 2024-12-01T14:29:44,488 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. after waiting 0 ms 2024-12-01T14:29:44,488 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. 2024-12-01T14:29:44,488 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. 2024-12-01T14:29:44,488 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 881998cd3b67297a80c91841da1e0396: Waiting for close lock at 1733063384488Disabling compacts and flushes for region at 1733063384488Disabling writes for close at 1733063384488Writing region close event to WAL at 1733063384488Closed at 1733063384488 2024-12-01T14:29:44,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741853_1029 (size=95) 2024-12-01T14:29:44,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741853_1029 (size=95) 2024-12-01T14:29:44,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741853_1029 (size=95) 2024-12-01T14:29:44,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:29:44,496 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-53100430:(num 1733063384438) 2024-12-01T14:29:44,496 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:44,499 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:44,513 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499, exclude list is [], retry=0 2024-12-01T14:29:44,517 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:44,517 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:44,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:44,520 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 2024-12-01T14:29:44,522 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:29:44,718 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499, size=0 (0bytes) 2024-12-01T14:29:44,718 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 might be still open, length is 0 2024-12-01T14:29:44,718 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 2024-12-01T14:29:44,719 WARN [IPC Server handler 4 on default port 40889 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-01T14:29:44,720 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 after 2ms 2024-12-01T14:29:45,496 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:52244 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33423:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52244 dst: /127.0.0.1:33423 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33423 remote=/127.0.0.1:52244]. Total timeout mills is 60000, 59191 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:29:45,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:34108 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33303:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34108 dst: /127.0.0.1:33303 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:29:45,497 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:54484 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40063:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54484 dst: /127.0.0.1:40063 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:29:45,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741855_1032 (size=263633) 2024-12-01T14:29:45,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741855_1032 (size=263633) 2024-12-01T14:29:45,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741855_1032 (size=263633) 2024-12-01T14:29:48,722 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 after 4004ms 2024-12-01T14:29:48,733 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:48,734 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 took 4016ms 2024-12-01T14:29:48,738 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733063384499.temp 2024-12-01T14:29:48,740 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000000001-wal.1733063384499.temp 2024-12-01T14:29:48,818 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T14:29:48,866 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499; continuing. 2024-12-01T14:29:48,866 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 so closing down 2024-12-01T14:29:48,866 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:48,867 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:48,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741856_1033 (size=263641) 2024-12-01T14:29:48,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741856_1033 (size=263641) 2024-12-01T14:29:48,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741856_1033 (size=263641) 2024-12-01T14:29:48,872 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000000001-wal.1733063384499.temp (wrote 3002 edits, skipped 0 edits in 67 ms) 2024-12-01T14:29:48,875 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000000001-wal.1733063384499.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000003002 2024-12-01T14:29:48,875 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 140 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T14:29:48,875 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499, journal: Splitting hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499, size=0 (0bytes) at 1733063384718Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000000001-wal.1733063384499.temp at 1733063388740 (+4022 ms)Split 1024 edits, skipped 0 edits. at 1733063388805 (+65 ms)Split 2048 edits, skipped 0 edits. at 1733063388837 (+32 ms)Finishing writing output for hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 so closing down at 1733063388866 (+29 ms)3 split writer threads finished at 1733063388867 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000000001-wal.1733063384499.temp (wrote 3002 edits, skipped 0 edits in 67 ms) at 1733063388872 (+5 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000000001-wal.1733063384499.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000003002 at 1733063388875 (+3 ms)Processed 3002 edits across 1 Regions in 140 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499, size=0, length=0, corrupted=false, cancelled=false at 1733063388875 2024-12-01T14:29:48,878 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 to hdfs://localhost:40889/hbase/oldWALs/wal.1733063384499 2024-12-01T14:29:48,880 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000003002 2024-12-01T14:29:48,880 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:48,883 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:48,886 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-01T14:29:48,898 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063388884, exclude list is [], retry=0 2024-12-01T14:29:48,901 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:48,901 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:48,901 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:48,903 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063388884 2024-12-01T14:29:48,904 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289)] 2024-12-01T14:29:48,904 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:48,906 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:48,907 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 881998cd3b67297a80c91841da1e0396 columnFamilyName a 2024-12-01T14:29:48,907 DEBUG [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:48,908 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(327): Store=881998cd3b67297a80c91841da1e0396/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:48,908 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:48,909 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 881998cd3b67297a80c91841da1e0396 columnFamilyName b 2024-12-01T14:29:48,909 DEBUG [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:48,910 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(327): Store=881998cd3b67297a80c91841da1e0396/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:48,910 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:48,911 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 881998cd3b67297a80c91841da1e0396 columnFamilyName c 2024-12-01T14:29:48,911 DEBUG [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:48,912 INFO [StoreOpener-881998cd3b67297a80c91841da1e0396-1 {}] regionserver.HStore(327): Store=881998cd3b67297a80c91841da1e0396/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:48,912 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:48,913 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:48,915 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:48,916 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000003002 2024-12-01T14:29:48,918 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:48,957 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T14:29:49,286 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 881998cd3b67297a80c91841da1e0396 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-01T14:29:49,319 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/a/194ddd85eb0e438e9a8d41f83c493647 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733063384533/Put/seqid=0 2024-12-01T14:29:49,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741858_1035 (size=50463) 2024-12-01T14:29:49,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741858_1035 (size=50463) 2024-12-01T14:29:49,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741858_1035 (size=50463) 2024-12-01T14:29:49,329 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/a/194ddd85eb0e438e9a8d41f83c493647 2024-12-01T14:29:49,338 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/a/194ddd85eb0e438e9a8d41f83c493647 as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/a/194ddd85eb0e438e9a8d41f83c493647 2024-12-01T14:29:49,347 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/a/194ddd85eb0e438e9a8d41f83c493647, entries=754, sequenceid=754, filesize=49.3 K 2024-12-01T14:29:49,347 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 881998cd3b67297a80c91841da1e0396 in 61ms, sequenceid=754, compaction requested=false; wal=null 2024-12-01T14:29:49,369 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T14:29:49,370 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 881998cd3b67297a80c91841da1e0396 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-01T14:29:49,377 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/a/289876ed6d9d44789d370fafc3eb6df9 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733063384578/Put/seqid=0 2024-12-01T14:29:49,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741859_1036 (size=20072) 2024-12-01T14:29:49,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741859_1036 (size=20072) 2024-12-01T14:29:49,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741859_1036 (size=20072) 2024-12-01T14:29:49,388 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/a/289876ed6d9d44789d370fafc3eb6df9 2024-12-01T14:29:49,413 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/b/8b2e1728465043778f8e19c69eb4acb8 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733063384601/Put/seqid=0 2024-12-01T14:29:49,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741860_1037 (size=35835) 2024-12-01T14:29:49,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741860_1037 (size=35835) 2024-12-01T14:29:49,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741860_1037 (size=35835) 2024-12-01T14:29:49,422 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/b/8b2e1728465043778f8e19c69eb4acb8 2024-12-01T14:29:49,429 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/a/289876ed6d9d44789d370fafc3eb6df9 as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/a/289876ed6d9d44789d370fafc3eb6df9 2024-12-01T14:29:49,435 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/a/289876ed6d9d44789d370fafc3eb6df9, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-01T14:29:49,437 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/b/8b2e1728465043778f8e19c69eb4acb8 as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/b/8b2e1728465043778f8e19c69eb4acb8 2024-12-01T14:29:49,444 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/b/8b2e1728465043778f8e19c69eb4acb8, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-01T14:29:49,444 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 881998cd3b67297a80c91841da1e0396 in 75ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-01T14:29:49,455 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T14:29:49,455 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 881998cd3b67297a80c91841da1e0396 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-01T14:29:49,462 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/b/6a39090665ea4bb8a6b7b83ce43015bf is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733063384625/Put/seqid=0 2024-12-01T14:29:49,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741861_1038 (size=35082) 2024-12-01T14:29:49,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741861_1038 (size=35082) 2024-12-01T14:29:49,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741861_1038 (size=35082) 2024-12-01T14:29:49,471 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/b/6a39090665ea4bb8a6b7b83ce43015bf 2024-12-01T14:29:49,492 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/c/57350e95e9ff4e7da0df9c18baf49185 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733063384648/Put/seqid=0 2024-12-01T14:29:49,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741862_1039 (size=20825) 2024-12-01T14:29:49,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741862_1039 (size=20825) 2024-12-01T14:29:49,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741862_1039 (size=20825) 2024-12-01T14:29:49,501 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/c/57350e95e9ff4e7da0df9c18baf49185 2024-12-01T14:29:49,510 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/b/6a39090665ea4bb8a6b7b83ce43015bf as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/b/6a39090665ea4bb8a6b7b83ce43015bf 2024-12-01T14:29:49,518 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/b/6a39090665ea4bb8a6b7b83ce43015bf, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-01T14:29:49,520 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/c/57350e95e9ff4e7da0df9c18baf49185 as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/c/57350e95e9ff4e7da0df9c18baf49185 2024-12-01T14:29:49,528 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/c/57350e95e9ff4e7da0df9c18baf49185, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-01T14:29:49,528 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 881998cd3b67297a80c91841da1e0396 in 73ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-01T14:29:49,546 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733063384686/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. 2024-12-01T14:29:49,550 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000003002 2024-12-01T14:29:49,551 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-01T14:29:49,551 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 881998cd3b67297a80c91841da1e0396 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-01T14:29:49,561 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/c/7e31eef7215e49a1879d6cd153ff0a8a is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733063384655/Put/seqid=0 2024-12-01T14:29:49,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741863_1040 (size=50301) 2024-12-01T14:29:49,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741863_1040 (size=50301) 2024-12-01T14:29:49,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741863_1040 (size=50301) 2024-12-01T14:29:49,571 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/c/7e31eef7215e49a1879d6cd153ff0a8a 2024-12-01T14:29:49,577 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7e31eef7215e49a1879d6cd153ff0a8a 2024-12-01T14:29:49,579 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/.tmp/c/7e31eef7215e49a1879d6cd153ff0a8a as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/c/7e31eef7215e49a1879d6cd153ff0a8a 2024-12-01T14:29:49,585 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7e31eef7215e49a1879d6cd153ff0a8a 2024-12-01T14:29:49,585 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/c/7e31eef7215e49a1879d6cd153ff0a8a, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-01T14:29:49,585 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 881998cd3b67297a80c91841da1e0396 in 34ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-01T14:29:49,586 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/0000000000000003002 2024-12-01T14:29:49,587 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:49,587 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:49,588 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-01T14:29:49,589 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 881998cd3b67297a80c91841da1e0396 2024-12-01T14:29:49,592 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenIntoWAL/881998cd3b67297a80c91841da1e0396/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-01T14:29:49,593 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 881998cd3b67297a80c91841da1e0396; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74904845, jitterRate=0.11616916954517365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-01T14:29:49,593 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 881998cd3b67297a80c91841da1e0396: Writing region info on filesystem at 1733063388904Initializing all the Stores at 1733063388905 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063388906 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063388906Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063388906Cleaning up temporary data from old regions at 1733063389587 (+681 ms)Region opened successfully at 1733063389593 (+6 ms) 2024-12-01T14:29:49,659 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 881998cd3b67297a80c91841da1e0396, disabling compactions & flushes 2024-12-01T14:29:49,659 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. 2024-12-01T14:29:49,659 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. 2024-12-01T14:29:49,659 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. after waiting 0 ms 2024-12-01T14:29:49,659 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. 2024-12-01T14:29:49,661 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733063384432.881998cd3b67297a80c91841da1e0396. 2024-12-01T14:29:49,661 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 881998cd3b67297a80c91841da1e0396: Waiting for close lock at 1733063389658Disabling compacts and flushes for region at 1733063389658Disabling writes for close at 1733063389659 (+1 ms)Writing region close event to WAL at 1733063389661 (+2 ms)Closed at 1733063389661 2024-12-01T14:29:49,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741857_1034 (size=95) 2024-12-01T14:29:49,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741857_1034 (size=95) 2024-12-01T14:29:49,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741857_1034 (size=95) 2024-12-01T14:29:49,667 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:29:49,667 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733063388884) 2024-12-01T14:29:49,682 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=404 (was 390) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:36509 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_359143697_22 at /127.0.0.1:52294 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42983 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:42983 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_359143697_22 at /127.0.0.1:54496 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_359143697_22 at /127.0.0.1:34178 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:40889 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_359143697_22 at /127.0.0.1:34144 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:40889 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36509 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=218 (was 219), ProcessCount=11 (was 11), AvailableMemoryMB=5240 (was 5306) 2024-12-01T14:29:49,695 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=218, ProcessCount=11, AvailableMemoryMB=5238 2024-12-01T14:29:49,711 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:49,713 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:49,714 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:49,718 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-51548576, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/hregion-51548576, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:49,737 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-51548576/hregion-51548576.1733063389718, exclude list is [], retry=0 2024-12-01T14:29:49,741 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:49,741 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:49,742 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:49,745 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-51548576/hregion-51548576.1733063389718 2024-12-01T14:29:49,745 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289)] 2024-12-01T14:29:49,745 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0b80415be4e2a301a281e22b1004a482, NAME => 'test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/hbase 2024-12-01T14:29:49,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741865_1042 (size=43) 2024-12-01T14:29:49,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741865_1042 (size=43) 2024-12-01T14:29:49,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741865_1042 (size=43) 2024-12-01T14:29:49,758 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:49,760 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:49,761 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b80415be4e2a301a281e22b1004a482 columnFamilyName a 2024-12-01T14:29:49,762 DEBUG [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:49,762 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(327): Store=0b80415be4e2a301a281e22b1004a482/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:49,763 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:49,764 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b80415be4e2a301a281e22b1004a482 columnFamilyName b 2024-12-01T14:29:49,765 DEBUG [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:49,765 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(327): Store=0b80415be4e2a301a281e22b1004a482/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:49,765 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:49,767 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b80415be4e2a301a281e22b1004a482 columnFamilyName c 2024-12-01T14:29:49,767 DEBUG [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:49,768 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(327): Store=0b80415be4e2a301a281e22b1004a482/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:49,768 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:49,769 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:49,770 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:49,771 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:49,771 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:49,772 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:49,773 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:49,775 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:49,776 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0b80415be4e2a301a281e22b1004a482; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61844210, jitterRate=-0.07844945788383484}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:49,777 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0b80415be4e2a301a281e22b1004a482: Writing region info on filesystem at 1733063389758Initializing all the Stores at 1733063389759 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063389759Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063389759Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063389759Cleaning up temporary data from old regions at 1733063389771 (+12 ms)Region opened successfully at 1733063389776 (+5 ms) 2024-12-01T14:29:49,777 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0b80415be4e2a301a281e22b1004a482, disabling compactions & flushes 2024-12-01T14:29:49,777 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. 2024-12-01T14:29:49,777 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. 2024-12-01T14:29:49,777 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. after waiting 0 ms 2024-12-01T14:29:49,777 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. 2024-12-01T14:29:49,777 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. 2024-12-01T14:29:49,778 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0b80415be4e2a301a281e22b1004a482: Waiting for close lock at 1733063389777Disabling compacts and flushes for region at 1733063389777Disabling writes for close at 1733063389777Writing region close event to WAL at 1733063389777Closed at 1733063389777 2024-12-01T14:29:49,781 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-51548576/hregion-51548576.1733063389718 not finished, retry = 0 2024-12-01T14:29:49,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741864_1041 (size=95) 2024-12-01T14:29:49,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741864_1041 (size=95) 2024-12-01T14:29:49,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741864_1041 (size=95) 2024-12-01T14:29:49,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:29:49,884 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-51548576:(num 1733063389718) 2024-12-01T14:29:49,885 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:49,887 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:49,905 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888, exclude list is [], retry=0 2024-12-01T14:29:49,908 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:49,909 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:49,909 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:49,911 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888 2024-12-01T14:29:49,911 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:40289:40289)] 2024-12-01T14:29:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741866_1043 (size=263359) 2024-12-01T14:29:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741866_1043 (size=263359) 2024-12-01T14:29:50,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741866_1043 (size=263359) 2024-12-01T14:29:50,097 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888, size=257.2 K (263359bytes) 2024-12-01T14:29:50,097 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888 2024-12-01T14:29:50,097 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888 after 0ms 2024-12-01T14:29:50,100 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:50,102 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888 took 6ms 2024-12-01T14:29:50,108 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733063389888.temp 2024-12-01T14:29:50,110 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000000001-wal.1733063389888.temp 2024-12-01T14:29:50,163 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888 so closing down 2024-12-01T14:29:50,163 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:50,163 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:50,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741867_1044 (size=263359) 2024-12-01T14:29:50,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741867_1044 (size=263359) 2024-12-01T14:29:50,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741867_1044 (size=263359) 2024-12-01T14:29:50,167 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000000001-wal.1733063389888.temp (wrote 3000 edits, skipped 0 edits in 41 ms) 2024-12-01T14:29:50,169 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000000001-wal.1733063389888.temp to hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003000 2024-12-01T14:29:50,169 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-12-01T14:29:50,169 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888, journal: Splitting hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888, size=257.2 K (263359bytes) at 1733063390097Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000000001-wal.1733063389888.temp at 1733063390110 (+13 ms)Split 1024 edits, skipped 0 edits. at 1733063390125 (+15 ms)Split 2048 edits, skipped 0 edits. at 1733063390145 (+20 ms)Finishing writing output for hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888 so closing down at 1733063390163 (+18 ms)3 split writer threads finished at 1733063390163Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000000001-wal.1733063389888.temp (wrote 3000 edits, skipped 0 edits in 41 ms) at 1733063390167 (+4 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000000001-wal.1733063389888.temp to hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003000 at 1733063390169 (+2 ms)Processed 3000 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1733063390169 2024-12-01T14:29:50,171 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063389888 to hdfs://localhost:40889/hbase/oldWALs/wal.1733063389888 2024-12-01T14:29:50,172 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003000 2024-12-01T14:29:50,173 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:50,175 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:50,190 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176, exclude list is [], retry=0 2024-12-01T14:29:50,193 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:50,193 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:50,193 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:50,195 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176 2024-12-01T14:29:50,195 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:29:50,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741868_1045 (size=263486) 2024-12-01T14:29:50,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741868_1045 (size=263486) 2024-12-01T14:29:50,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741868_1045 (size=263486) 2024-12-01T14:29:50,398 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176, size=257.3 K (263486bytes) 2024-12-01T14:29:50,398 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176 2024-12-01T14:29:50,398 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176 after 0ms 2024-12-01T14:29:50,401 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:50,402 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176 took 5ms 2024-12-01T14:29:50,406 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733063390176.temp 2024-12-01T14:29:50,408 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003001-wal.1733063390176.temp 2024-12-01T14:29:50,453 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176 so closing down 2024-12-01T14:29:50,453 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:50,453 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:50,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741869_1046 (size=263486) 2024-12-01T14:29:50,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741869_1046 (size=263486) 2024-12-01T14:29:50,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741869_1046 (size=263486) 2024-12-01T14:29:50,457 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003001-wal.1733063390176.temp (wrote 3000 edits, skipped 0 edits in 34 ms) 2024-12-01T14:29:50,458 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003001-wal.1733063390176.temp to hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000006000 2024-12-01T14:29:50,459 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-12-01T14:29:50,459 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176, journal: Splitting hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176, size=257.3 K (263486bytes) at 1733063390398Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003001-wal.1733063390176.temp at 1733063390408 (+10 ms)Split 1024 edits, skipped 0 edits. at 1733063390423 (+15 ms)Split 2048 edits, skipped 0 edits. at 1733063390439 (+16 ms)Finishing writing output for hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176 so closing down at 1733063390453 (+14 ms)3 split writer threads finished at 1733063390453Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003001-wal.1733063390176.temp (wrote 3000 edits, skipped 0 edits in 34 ms) at 1733063390457 (+4 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003001-wal.1733063390176.temp to hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000006000 at 1733063390459 (+2 ms)Processed 3000 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1733063390459 2024-12-01T14:29:50,461 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390176 to hdfs://localhost:40889/hbase/oldWALs/wal.1733063390176 2024-12-01T14:29:50,462 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000006000 2024-12-01T14:29:50,462 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:50,465 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/test2727-manual,16010,1733063389710, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:50,487 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390466, exclude list is [], retry=0 2024-12-01T14:29:50,490 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:50,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:50,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:50,494 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733063389710/wal.1733063390466 2024-12-01T14:29:50,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:40289:40289)] 2024-12-01T14:29:50,495 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b80415be4e2a301a281e22b1004a482, NAME => 'test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:50,495 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:50,495 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,495 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,497 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,499 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b80415be4e2a301a281e22b1004a482 columnFamilyName a 2024-12-01T14:29:50,499 DEBUG [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:50,500 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(327): Store=0b80415be4e2a301a281e22b1004a482/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:50,500 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,501 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b80415be4e2a301a281e22b1004a482 columnFamilyName b 2024-12-01T14:29:50,501 DEBUG [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:50,502 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(327): Store=0b80415be4e2a301a281e22b1004a482/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:50,502 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,504 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b80415be4e2a301a281e22b1004a482 columnFamilyName c 2024-12-01T14:29:50,504 DEBUG [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:50,505 INFO [StoreOpener-0b80415be4e2a301a281e22b1004a482-1 {}] regionserver.HStore(327): Store=0b80415be4e2a301a281e22b1004a482/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:50,505 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,506 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,508 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,509 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003000 2024-12-01T14:29:50,511 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:50,550 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003000 2024-12-01T14:29:50,552 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000006000 2024-12-01T14:29:50,554 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:50,594 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000006000 2024-12-01T14:29:50,594 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0b80415be4e2a301a281e22b1004a482 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-01T14:29:50,613 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/.tmp/a/2e42975d25ce4fc1b535552b4e85a042 is 41, key is test2727/a:100/1733063390199/Put/seqid=0 2024-12-01T14:29:50,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741871_1048 (size=84227) 2024-12-01T14:29:50,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741871_1048 (size=84227) 2024-12-01T14:29:50,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741871_1048 (size=84227) 2024-12-01T14:29:50,624 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/.tmp/a/2e42975d25ce4fc1b535552b4e85a042 2024-12-01T14:29:50,652 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/.tmp/b/2ba8291ccfdd4bd08f64c5149c86682a is 41, key is test2727/b:100/1733063390261/Put/seqid=0 2024-12-01T14:29:50,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741872_1049 (size=84609) 2024-12-01T14:29:50,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741872_1049 (size=84609) 2024-12-01T14:29:50,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741872_1049 (size=84609) 2024-12-01T14:29:50,659 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/.tmp/b/2ba8291ccfdd4bd08f64c5149c86682a 2024-12-01T14:29:50,692 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/.tmp/c/1b275f0388ea40349f68a9caa7557048 is 41, key is test2727/c:100/1733063390326/Put/seqid=0 2024-12-01T14:29:50,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741873_1050 (size=84609) 2024-12-01T14:29:50,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741873_1050 (size=84609) 2024-12-01T14:29:50,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741873_1050 (size=84609) 2024-12-01T14:29:50,700 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/.tmp/c/1b275f0388ea40349f68a9caa7557048 2024-12-01T14:29:50,708 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/.tmp/a/2e42975d25ce4fc1b535552b4e85a042 as hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/a/2e42975d25ce4fc1b535552b4e85a042 2024-12-01T14:29:50,716 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/a/2e42975d25ce4fc1b535552b4e85a042, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-01T14:29:50,717 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/.tmp/b/2ba8291ccfdd4bd08f64c5149c86682a as hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/b/2ba8291ccfdd4bd08f64c5149c86682a 2024-12-01T14:29:50,724 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/b/2ba8291ccfdd4bd08f64c5149c86682a, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-01T14:29:50,726 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/.tmp/c/1b275f0388ea40349f68a9caa7557048 as hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/c/1b275f0388ea40349f68a9caa7557048 2024-12-01T14:29:50,731 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-01T14:29:50,731 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-01T14:29:50,732 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T14:29:50,732 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-01T14:29:50,733 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-01T14:29:50,733 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-01T14:29:50,733 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/c/1b275f0388ea40349f68a9caa7557048, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-01T14:29:50,733 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 0b80415be4e2a301a281e22b1004a482 in 139ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-01T14:29:50,733 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-01T14:29:50,733 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-01T14:29:50,734 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000003000 2024-12-01T14:29:50,735 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/0000000000000006000 2024-12-01T14:29:50,737 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,737 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,738 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:50,739 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0b80415be4e2a301a281e22b1004a482 2024-12-01T14:29:50,742 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/test2727/0b80415be4e2a301a281e22b1004a482/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-01T14:29:50,743 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0b80415be4e2a301a281e22b1004a482; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74459350, jitterRate=0.10953077673912048}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:50,744 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0b80415be4e2a301a281e22b1004a482: Writing region info on filesystem at 1733063390496Initializing all the Stores at 1733063390497 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063390497Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063390497Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063390497Obtaining lock to block concurrent updates at 1733063390594 (+97 ms)Preparing flush snapshotting stores in 0b80415be4e2a301a281e22b1004a482 at 1733063390594Finished memstore snapshotting test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733063390594Flushing stores of test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. at 1733063390594Flushing 0b80415be4e2a301a281e22b1004a482/a: creating writer at 1733063390594Flushing 0b80415be4e2a301a281e22b1004a482/a: appending metadata at 1733063390613 (+19 ms)Flushing 0b80415be4e2a301a281e22b1004a482/a: closing flushed file at 1733063390613Flushing 0b80415be4e2a301a281e22b1004a482/b: creating writer at 1733063390632 (+19 ms)Flushing 0b80415be4e2a301a281e22b1004a482/b: appending metadata at 1733063390650 (+18 ms)Flushing 0b80415be4e2a301a281e22b1004a482/b: closing flushed file at 1733063390650Flushing 0b80415be4e2a301a281e22b1004a482/c: creating writer at 1733063390666 (+16 ms)Flushing 0b80415be4e2a301a281e22b1004a482/c: appending metadata at 1733063390691 (+25 ms)Flushing 0b80415be4e2a301a281e22b1004a482/c: closing flushed file at 1733063390691Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba81efe: reopening flushed file at 1733063390707 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@231c1eea: reopening flushed file at 1733063390716 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@643e2d4f: reopening flushed file at 1733063390725 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 0b80415be4e2a301a281e22b1004a482 in 139ms, sequenceid=6000, compaction requested=false; wal=null at 1733063390733 (+8 ms)Cleaning up temporary data from old regions at 1733063390737 (+4 ms)Region opened successfully at 1733063390744 (+7 ms) 2024-12-01T14:29:50,745 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-01T14:29:50,745 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0b80415be4e2a301a281e22b1004a482, disabling compactions & flushes 2024-12-01T14:29:50,745 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. 2024-12-01T14:29:50,746 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. 2024-12-01T14:29:50,746 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. after waiting 0 ms 2024-12-01T14:29:50,746 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. 2024-12-01T14:29:50,747 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733063389712.0b80415be4e2a301a281e22b1004a482. 2024-12-01T14:29:50,747 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0b80415be4e2a301a281e22b1004a482: Waiting for close lock at 1733063390745Disabling compacts and flushes for region at 1733063390745Disabling writes for close at 1733063390746 (+1 ms)Writing region close event to WAL at 1733063390747 (+1 ms)Closed at 1733063390747 2024-12-01T14:29:50,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741870_1047 (size=95) 2024-12-01T14:29:50,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741870_1047 (size=95) 2024-12-01T14:29:50,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741870_1047 (size=95) 2024-12-01T14:29:50,753 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:29:50,753 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733063390466) 2024-12-01T14:29:50,767 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=406 (was 404) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:52396 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:34144 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:54662 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=218 (was 218), ProcessCount=11 (was 11), AvailableMemoryMB=5035 (was 5238) 2024-12-01T14:29:50,780 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=406, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=218, ProcessCount=11, AvailableMemoryMB=5035 2024-12-01T14:29:50,802 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:50,810 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:50,810 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733063390810 2024-12-01T14:29:50,818 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810 2024-12-01T14:29:50,820 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:29:50,821 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 06bd89eb33d0c3f3f019b97badaddcb8, NAME => 'testSequentialEditLogSeqNum,,1733063390803.06bd89eb33d0c3f3f019b97badaddcb8.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:50,821 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733063390803.06bd89eb33d0c3f3f019b97badaddcb8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:50,821 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 06bd89eb33d0c3f3f019b97badaddcb8 2024-12-01T14:29:50,821 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 06bd89eb33d0c3f3f019b97badaddcb8 2024-12-01T14:29:50,822 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8 doesn't exist for region: 06bd89eb33d0c3f3f019b97badaddcb8 on table testSequentialEditLogSeqNum 2024-12-01T14:29:50,822 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 06bd89eb33d0c3f3f019b97badaddcb8 on table testSequentialEditLogSeqNum 2024-12-01T14:29:50,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741875_1052 (size=62) 2024-12-01T14:29:50,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741875_1052 (size=62) 2024-12-01T14:29:50,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741875_1052 (size=62) 2024-12-01T14:29:50,834 INFO [StoreOpener-06bd89eb33d0c3f3f019b97badaddcb8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 06bd89eb33d0c3f3f019b97badaddcb8 2024-12-01T14:29:50,836 INFO [StoreOpener-06bd89eb33d0c3f3f019b97badaddcb8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06bd89eb33d0c3f3f019b97badaddcb8 columnFamilyName a 2024-12-01T14:29:50,836 DEBUG [StoreOpener-06bd89eb33d0c3f3f019b97badaddcb8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:50,836 INFO [StoreOpener-06bd89eb33d0c3f3f019b97badaddcb8-1 {}] regionserver.HStore(327): Store=06bd89eb33d0c3f3f019b97badaddcb8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:50,837 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 06bd89eb33d0c3f3f019b97badaddcb8 2024-12-01T14:29:50,837 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8 2024-12-01T14:29:50,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8 2024-12-01T14:29:50,838 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 06bd89eb33d0c3f3f019b97badaddcb8 2024-12-01T14:29:50,838 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 06bd89eb33d0c3f3f019b97badaddcb8 2024-12-01T14:29:50,840 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 06bd89eb33d0c3f3f019b97badaddcb8 2024-12-01T14:29:50,843 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:50,843 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 06bd89eb33d0c3f3f019b97badaddcb8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66708711, jitterRate=-0.005962744355201721}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:29:50,844 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 06bd89eb33d0c3f3f019b97badaddcb8: Writing region info on filesystem at 1733063390821Initializing all the Stores at 1733063390834 (+13 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063390834Cleaning up temporary data from old regions at 1733063390838 (+4 ms)Region opened successfully at 1733063390844 (+6 ms) 2024-12-01T14:29:50,858 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 06bd89eb33d0c3f3f019b97badaddcb8 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-01T14:29:50,879 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/.tmp/a/d0ccebe06e0a49a38361f950e91a77a9 is 81, key is testSequentialEditLogSeqNum/a:x0/1733063390845/Put/seqid=0 2024-12-01T14:29:50,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741876_1053 (size=5833) 2024-12-01T14:29:50,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741876_1053 (size=5833) 2024-12-01T14:29:50,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741876_1053 (size=5833) 2024-12-01T14:29:50,889 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/.tmp/a/d0ccebe06e0a49a38361f950e91a77a9 2024-12-01T14:29:50,898 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/.tmp/a/d0ccebe06e0a49a38361f950e91a77a9 as hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/a/d0ccebe06e0a49a38361f950e91a77a9 2024-12-01T14:29:50,906 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/a/d0ccebe06e0a49a38361f950e91a77a9, entries=10, sequenceid=13, filesize=5.7 K 2024-12-01T14:29:50,908 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 06bd89eb33d0c3f3f019b97badaddcb8 in 50ms, sequenceid=13, compaction requested=false 2024-12-01T14:29:50,908 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 06bd89eb33d0c3f3f019b97badaddcb8: 2024-12-01T14:29:50,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T14:29:50,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T14:29:50,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T14:29:50,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T14:29:50,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-01T14:29:50,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741874_1051 (size=1843) 2024-12-01T14:29:50,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741874_1051 (size=1843) 2024-12-01T14:29:50,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741874_1051 (size=1843) 2024-12-01T14:29:50,932 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810, size=1.8 K (1843bytes) 2024-12-01T14:29:50,932 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810 2024-12-01T14:29:50,933 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810 after 1ms 2024-12-01T14:29:50,935 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:50,936 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810 took 4ms 2024-12-01T14:29:50,939 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810 so closing down 2024-12-01T14:29:50,939 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:50,940 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733063390810.temp 2024-12-01T14:29:50,942 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/recovered.edits/0000000000000000003-wal.1733063390810.temp 2024-12-01T14:29:50,943 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:50,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741877_1054 (size=1477) 2024-12-01T14:29:50,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741877_1054 (size=1477) 2024-12-01T14:29:50,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741877_1054 (size=1477) 2024-12-01T14:29:50,955 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/recovered.edits/0000000000000000003-wal.1733063390810.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-01T14:29:50,957 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/recovered.edits/0000000000000000003-wal.1733063390810.temp to hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/recovered.edits/0000000000000000020 2024-12-01T14:29:50,957 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810, size=1.8 K, length=1843, corrupted=false, cancelled=false 2024-12-01T14:29:50,957 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810, journal: Splitting hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810, size=1.8 K (1843bytes) at 1733063390932Finishing writing output for hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810 so closing down at 1733063390939 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/recovered.edits/0000000000000000003-wal.1733063390810.temp at 1733063390942 (+3 ms)3 split writer threads finished at 1733063390943 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/recovered.edits/0000000000000000003-wal.1733063390810.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733063390955 (+12 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/recovered.edits/0000000000000000003-wal.1733063390810.temp to hdfs://localhost:40889/hbase/data/default/testSequentialEditLogSeqNum/06bd89eb33d0c3f3f019b97badaddcb8/recovered.edits/0000000000000000020 at 1733063390957 (+2 ms)Processed 17 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:40889/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733063390802/wal.1733063390810, size=1.8 K, length=1843, corrupted=false, cancelled=false at 1733063390957 2024-12-01T14:29:50,973 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=411 (was 406) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:52396 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:34144 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=218 (was 218), ProcessCount=11 (was 11), AvailableMemoryMB=5009 (was 5035) 2024-12-01T14:29:50,986 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=411, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=218, ProcessCount=11, AvailableMemoryMB=5006 2024-12-01T14:29:51,025 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:51,030 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:51,073 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:51,076 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26267985, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/hregion-26267985, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:51,093 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26267985/hregion-26267985.1733063391076, exclude list is [], retry=0 2024-12-01T14:29:51,096 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:51,096 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:51,101 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:51,106 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26267985/hregion-26267985.1733063391076 2024-12-01T14:29:51,107 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289)] 2024-12-01T14:29:51,107 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3661663ccd6d161f9484e66230b3fccd, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/hbase 2024-12-01T14:29:51,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741879_1056 (size=70) 2024-12-01T14:29:51,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741879_1056 (size=70) 2024-12-01T14:29:51,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741879_1056 (size=70) 2024-12-01T14:29:51,120 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:51,122 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,123 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3661663ccd6d161f9484e66230b3fccd columnFamilyName a 2024-12-01T14:29:51,124 DEBUG [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:51,124 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(327): Store=3661663ccd6d161f9484e66230b3fccd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:51,124 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,126 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3661663ccd6d161f9484e66230b3fccd columnFamilyName b 2024-12-01T14:29:51,126 DEBUG [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:51,126 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(327): Store=3661663ccd6d161f9484e66230b3fccd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:51,127 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,128 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3661663ccd6d161f9484e66230b3fccd columnFamilyName c 2024-12-01T14:29:51,128 DEBUG [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:51,129 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(327): Store=3661663ccd6d161f9484e66230b3fccd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:51,129 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,130 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,130 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,131 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,131 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,132 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:51,133 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,141 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:51,141 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3661663ccd6d161f9484e66230b3fccd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72894804, jitterRate=0.08621722459793091}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:51,143 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3661663ccd6d161f9484e66230b3fccd: Writing region info on filesystem at 1733063391120Initializing all the Stores at 1733063391121 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063391121Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063391121Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063391121Cleaning up temporary data from old regions at 1733063391131 (+10 ms)Region opened successfully at 1733063391143 (+12 ms) 2024-12-01T14:29:51,143 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3661663ccd6d161f9484e66230b3fccd, disabling compactions & flushes 2024-12-01T14:29:51,143 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. 2024-12-01T14:29:51,143 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. 2024-12-01T14:29:51,143 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. after waiting 0 ms 2024-12-01T14:29:51,143 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. 2024-12-01T14:29:51,144 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. 2024-12-01T14:29:51,144 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3661663ccd6d161f9484e66230b3fccd: Waiting for close lock at 1733063391143Disabling compacts and flushes for region at 1733063391143Disabling writes for close at 1733063391143Writing region close event to WAL at 1733063391144 (+1 ms)Closed at 1733063391144 2024-12-01T14:29:51,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741878_1055 (size=95) 2024-12-01T14:29:51,147 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-26267985/hregion-26267985.1733063391076 not finished, retry = 0 2024-12-01T14:29:51,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741878_1055 (size=95) 2024-12-01T14:29:51,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741878_1055 (size=95) 2024-12-01T14:29:51,251 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:29:51,251 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26267985:(num 1733063391076) 2024-12-01T14:29:51,251 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:51,254 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:51,269 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254, exclude list is [], retry=0 2024-12-01T14:29:51,272 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:51,272 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:51,272 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:51,274 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 2024-12-01T14:29:51,275 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289)] 2024-12-01T14:29:51,275 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3661663ccd6d161f9484e66230b3fccd, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:51,275 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:51,275 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,275 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,276 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,277 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3661663ccd6d161f9484e66230b3fccd columnFamilyName a 2024-12-01T14:29:51,277 DEBUG [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:51,278 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(327): Store=3661663ccd6d161f9484e66230b3fccd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:51,278 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,279 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3661663ccd6d161f9484e66230b3fccd columnFamilyName b 2024-12-01T14:29:51,279 DEBUG [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:51,279 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(327): Store=3661663ccd6d161f9484e66230b3fccd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:51,279 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,280 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3661663ccd6d161f9484e66230b3fccd columnFamilyName c 2024-12-01T14:29:51,280 DEBUG [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:51,281 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(327): Store=3661663ccd6d161f9484e66230b3fccd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:51,281 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,281 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,283 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,284 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,284 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,284 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:51,286 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:51,287 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3661663ccd6d161f9484e66230b3fccd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70617468, jitterRate=0.05228227376937866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:51,287 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3661663ccd6d161f9484e66230b3fccd: Writing region info on filesystem at 1733063391275Initializing all the Stores at 1733063391276 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063391276Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063391276Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063391276Cleaning up temporary data from old regions at 1733063391284 (+8 ms)Region opened successfully at 1733063391287 (+3 ms) 2024-12-01T14:29:51,291 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733063391290/Put/seqid=0 2024-12-01T14:29:51,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741881_1058 (size=4826) 2024-12-01T14:29:51,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741881_1058 (size=4826) 2024-12-01T14:29:51,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741881_1058 (size=4826) 2024-12-01T14:29:51,298 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40889/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 3661663ccd6d161f9484e66230b3fccd/a 2024-12-01T14:29:51,306 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-01T14:29:51,306 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T14:29:51,306 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3661663ccd6d161f9484e66230b3fccd: 2024-12-01T14:29:51,308 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/a/9d5d9a89126d40739ae06e19e416218e_SeqId_3_ 2024-12-01T14:29:51,309 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40889/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 3661663ccd6d161f9484e66230b3fccd/a as hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/a/9d5d9a89126d40739ae06e19e416218e_SeqId_3_ - updating store file list. 2024-12-01T14:29:51,315 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9d5d9a89126d40739ae06e19e416218e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-01T14:29:51,315 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/a/9d5d9a89126d40739ae06e19e416218e_SeqId_3_ into 3661663ccd6d161f9484e66230b3fccd/a 2024-12-01T14:29:51,315 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40889/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 3661663ccd6d161f9484e66230b3fccd/a (new location: hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/a/9d5d9a89126d40739ae06e19e416218e_SeqId_3_) 2024-12-01T14:29:51,348 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254, size=0 (0bytes) 2024-12-01T14:29:51,348 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 might be still open, length is 0 2024-12-01T14:29:51,348 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 2024-12-01T14:29:51,349 WARN [IPC Server handler 2 on default port 40889 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-01T14:29:51,349 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 after 1ms 2024-12-01T14:29:51,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:54766 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40063:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54766 dst: /127.0.0.1:40063 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40063 remote=/127.0.0.1:54766]. Total timeout mills is 60000, 59832 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:29:51,491 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:52474 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33423:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52474 dst: /127.0.0.1:33423 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:29:51,491 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:34412 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33303:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34412 dst: /127.0.0.1:33303 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:29:51,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741880_1059 (size=472) 2024-12-01T14:29:51,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741880_1059 (size=472) 2024-12-01T14:29:55,350 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 after 4002ms 2024-12-01T14:29:55,354 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:55,355 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 took 4006ms 2024-12-01T14:29:55,358 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254; continuing. 2024-12-01T14:29:55,358 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 so closing down 2024-12-01T14:29:55,358 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:55,361 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733063391254.temp 2024-12-01T14:29:55,363 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005-wal.1733063391254.temp 2024-12-01T14:29:55,363 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:55,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741882_1060 (size=259) 2024-12-01T14:29:55,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741882_1060 (size=259) 2024-12-01T14:29:55,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741882_1060 (size=259) 2024-12-01T14:29:55,371 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005-wal.1733063391254.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-01T14:29:55,373 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005-wal.1733063391254.temp to hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005 2024-12-01T14:29:55,373 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T14:29:55,373 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254, journal: Splitting hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254, size=0 (0bytes) at 1733063391348Finishing writing output for hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 so closing down at 1733063395358 (+4010 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005-wal.1733063391254.temp at 1733063395363 (+5 ms)3 split writer threads finished at 1733063395363Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005-wal.1733063391254.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733063395371 (+8 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005-wal.1733063391254.temp to hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005 at 1733063395373 (+2 ms)Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254, size=0, length=0, corrupted=false, cancelled=false at 1733063395373 2024-12-01T14:29:55,375 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 to hdfs://localhost:40889/hbase/oldWALs/wal.1733063391254 2024-12-01T14:29:55,376 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005 2024-12-01T14:29:55,376 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:55,378 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:55,394 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063395379, exclude list is [], retry=0 2024-12-01T14:29:55,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:55,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:55,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:55,399 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063395379 2024-12-01T14:29:55,399 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:29:55,399 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3661663ccd6d161f9484e66230b3fccd, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:55,400 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:55,400 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,400 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,401 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,402 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3661663ccd6d161f9484e66230b3fccd columnFamilyName a 2024-12-01T14:29:55,402 DEBUG [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:55,409 DEBUG [StoreFileOpener-3661663ccd6d161f9484e66230b3fccd-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 9d5d9a89126d40739ae06e19e416218e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-01T14:29:55,409 DEBUG [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/a/9d5d9a89126d40739ae06e19e416218e_SeqId_3_ 2024-12-01T14:29:55,409 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(327): Store=3661663ccd6d161f9484e66230b3fccd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:55,410 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,411 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3661663ccd6d161f9484e66230b3fccd columnFamilyName b 2024-12-01T14:29:55,411 DEBUG [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:55,411 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(327): Store=3661663ccd6d161f9484e66230b3fccd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:55,411 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,412 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3661663ccd6d161f9484e66230b3fccd columnFamilyName c 2024-12-01T14:29:55,412 DEBUG [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:55,413 INFO [StoreOpener-3661663ccd6d161f9484e66230b3fccd-1 {}] regionserver.HStore(327): Store=3661663ccd6d161f9484e66230b3fccd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:55,413 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,414 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,416 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,416 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005 2024-12-01T14:29:55,418 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:55,419 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005 2024-12-01T14:29:55,419 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3661663ccd6d161f9484e66230b3fccd 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-01T14:29:55,436 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/.tmp/a/e5672aa4683c4668a1d68ad7971ca35c is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733063391319/Put/seqid=0 2024-12-01T14:29:55,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741884_1062 (size=5149) 2024-12-01T14:29:55,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741884_1062 (size=5149) 2024-12-01T14:29:55,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741884_1062 (size=5149) 2024-12-01T14:29:55,443 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/.tmp/a/e5672aa4683c4668a1d68ad7971ca35c 2024-12-01T14:29:55,449 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/.tmp/a/e5672aa4683c4668a1d68ad7971ca35c as hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/a/e5672aa4683c4668a1d68ad7971ca35c 2024-12-01T14:29:55,454 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/a/e5672aa4683c4668a1d68ad7971ca35c, entries=1, sequenceid=5, filesize=5.0 K 2024-12-01T14:29:55,454 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 3661663ccd6d161f9484e66230b3fccd in 35ms, sequenceid=5, compaction requested=false; wal=null 2024-12-01T14:29:55,455 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/0000000000000000005 2024-12-01T14:29:55,456 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,456 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,457 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:55,458 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3661663ccd6d161f9484e66230b3fccd 2024-12-01T14:29:55,461 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3661663ccd6d161f9484e66230b3fccd/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-01T14:29:55,462 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3661663ccd6d161f9484e66230b3fccd; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64112607, jitterRate=-0.0446477085351944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:55,462 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3661663ccd6d161f9484e66230b3fccd: Writing region info on filesystem at 1733063395400Initializing all the Stores at 1733063395401 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063395401Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063395401Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063395401Obtaining lock to block concurrent updates at 1733063395419 (+18 ms)Preparing flush snapshotting stores in 3661663ccd6d161f9484e66230b3fccd at 1733063395419Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733063395419Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. at 1733063395419Flushing 3661663ccd6d161f9484e66230b3fccd/a: creating writer at 1733063395420 (+1 ms)Flushing 3661663ccd6d161f9484e66230b3fccd/a: appending metadata at 1733063395435 (+15 ms)Flushing 3661663ccd6d161f9484e66230b3fccd/a: closing flushed file at 1733063395435Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67ae19cb: reopening flushed file at 1733063395448 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 3661663ccd6d161f9484e66230b3fccd in 35ms, sequenceid=5, compaction requested=false; wal=null at 1733063395454 (+6 ms)Cleaning up temporary data from old regions at 1733063395456 (+2 ms)Region opened successfully at 1733063395462 (+6 ms) 2024-12-01T14:29:55,467 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3661663ccd6d161f9484e66230b3fccd, disabling compactions & flushes 2024-12-01T14:29:55,467 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. 2024-12-01T14:29:55,467 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. 2024-12-01T14:29:55,467 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. after waiting 0 ms 2024-12-01T14:29:55,467 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. 2024-12-01T14:29:55,468 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733063391026.3661663ccd6d161f9484e66230b3fccd. 2024-12-01T14:29:55,468 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3661663ccd6d161f9484e66230b3fccd: Waiting for close lock at 1733063395467Disabling compacts and flushes for region at 1733063395467Disabling writes for close at 1733063395467Writing region close event to WAL at 1733063395468 (+1 ms)Closed at 1733063395468 2024-12-01T14:29:55,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741883_1061 (size=95) 2024-12-01T14:29:55,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741883_1061 (size=95) 2024-12-01T14:29:55,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741883_1061 (size=95) 2024-12-01T14:29:55,473 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:29:55,473 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733063395379) 2024-12-01T14:29:55,485 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=420 (was 411) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44683 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:40889 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:40889 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:44683 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1437649901_22 at /127.0.0.1:55938 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1437649901_22 at /127.0.0.1:41016 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1437649901_22 at /127.0.0.1:54116 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=200 (was 218), ProcessCount=11 (was 11), AvailableMemoryMB=4954 (was 5006) 2024-12-01T14:29:55,486 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-01T14:29:55,497 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=420, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=200, ProcessCount=11, AvailableMemoryMB=4953 2024-12-01T14:29:55,497 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-12-01T14:29:55,518 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:55,523 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T14:29:55,527 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 57b748500ef9,34327,1733063380396 2024-12-01T14:29:55,529 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4bf7101f 2024-12-01T14:29:55,530 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T14:29:55,532 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T14:29:55,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T14:29:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-01T14:29:55,543 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T14:29:55,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-01T14:29:55,545 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:55,547 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T14:29:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:29:55,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741885_1063 (size=694) 2024-12-01T14:29:55,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741885_1063 (size=694) 2024-12-01T14:29:55,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741885_1063 (size=694) 2024-12-01T14:29:55,563 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 44c73b6c4a612bcfc6cf8a6997a1cd8e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9 2024-12-01T14:29:55,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741886_1064 (size=77) 2024-12-01T14:29:55,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741886_1064 (size=77) 2024-12-01T14:29:55,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741886_1064 (size=77) 2024-12-01T14:29:55,575 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:55,575 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 44c73b6c4a612bcfc6cf8a6997a1cd8e, disabling compactions & flushes 2024-12-01T14:29:55,575 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:55,575 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:55,575 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. after waiting 0 ms 2024-12-01T14:29:55,575 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:55,575 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:55,575 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Waiting for close lock at 1733063395575Disabling compacts and flushes for region at 1733063395575Disabling writes for close at 1733063395575Writing region close event to WAL at 1733063395575Closed at 1733063395575 2024-12-01T14:29:55,577 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T14:29:55,584 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733063395577"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733063395577"}]},"ts":"1733063395577"} 2024-12-01T14:29:55,587 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T14:29:55,588 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T14:29:55,591 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733063395588"}]},"ts":"1733063395588"} 2024-12-01T14:29:55,594 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-01T14:29:55,595 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {57b748500ef9=0} racks are {/default-rack=0} 2024-12-01T14:29:55,596 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T14:29:55,596 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T14:29:55,596 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-01T14:29:55,596 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T14:29:55,596 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T14:29:55,596 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-01T14:29:55,596 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T14:29:55,596 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T14:29:55,596 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-01T14:29:55,596 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T14:29:55,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, ASSIGN}] 2024-12-01T14:29:55,600 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, ASSIGN 2024-12-01T14:29:55,602 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, ASSIGN; state=OFFLINE, location=57b748500ef9,36193,1733063381174; forceNewPlan=false, retain=false 2024-12-01T14:29:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:29:55,755 INFO [57b748500ef9:34327 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T14:29:55,757 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=OPENING, regionLocation=57b748500ef9,36193,1733063381174 2024-12-01T14:29:55,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, ASSIGN because future has completed 2024-12-01T14:29:55,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,36193,1733063381174}] 2024-12-01T14:29:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:29:55,919 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T14:29:55,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47979, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T14:29:55,934 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:55,934 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 44c73b6c4a612bcfc6cf8a6997a1cd8e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:55,934 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,935 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:55,935 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,935 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,937 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,940 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf1 2024-12-01T14:29:55,940 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:55,941 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:55,941 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,944 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf2 2024-12-01T14:29:55,944 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:55,945 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:55,945 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,946 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,946 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,947 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,947 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,948 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T14:29:55,949 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,951 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:55,952 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 44c73b6c4a612bcfc6cf8a6997a1cd8e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63178051, jitterRate=-0.05857367813587189}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T14:29:55,952 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:55,952 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Running coprocessor pre-open hook at 1733063395935Writing region info on filesystem at 1733063395935Initializing all the Stores at 1733063395936 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063395937 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063395937Cleaning up temporary data from old regions at 1733063395947 (+10 ms)Running coprocessor post-open hooks at 1733063395952 (+5 ms)Region opened successfully at 1733063395952 2024-12-01T14:29:55,954 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., pid=6, masterSystemTime=1733063395918 2024-12-01T14:29:55,957 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:55,957 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:55,958 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=OPEN, openSeqNum=2, regionLocation=57b748500ef9,36193,1733063381174 2024-12-01T14:29:55,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,36193,1733063381174 because future has completed 2024-12-01T14:29:55,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-01T14:29:55,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,36193,1733063381174 in 200 msec 2024-12-01T14:29:55,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-01T14:29:55,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, ASSIGN in 367 msec 2024-12-01T14:29:55,970 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T14:29:55,971 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733063395970"}]},"ts":"1733063395970"} 2024-12-01T14:29:55,973 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-01T14:29:55,974 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T14:29:55,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 437 msec 2024-12-01T14:29:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T14:29:56,178 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-01T14:29:56,178 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-01T14:29:56,180 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T14:29:56,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-01T14:29:56,190 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T14:29:56,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-01T14:29:56,203 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=2] 2024-12-01T14:29:56,204 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T14:29:56,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T14:29:56,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=44c73b6c4a612bcfc6cf8a6997a1cd8e, source=57b748500ef9,36193,1733063381174, destination=57b748500ef9,38129,1733063381264, warming up region on 57b748500ef9,38129,1733063381264 2024-12-01T14:29:56,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=44c73b6c4a612bcfc6cf8a6997a1cd8e, source=57b748500ef9,36193,1733063381174, destination=57b748500ef9,38129,1733063381264, running balancer 2024-12-01T14:29:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE 2024-12-01T14:29:56,224 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE 2024-12-01T14:29:56,226 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=CLOSING, regionLocation=57b748500ef9,36193,1733063381174 2024-12-01T14:29:56,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] regionserver.HRegion(7855): Warmup {ENCODED => 44c73b6c4a612bcfc6cf8a6997a1cd8e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:56,228 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,229 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf1 2024-12-01T14:29:56,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE because future has completed 2024-12-01T14:29:56,229 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:56,230 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-01T14:29:56,230 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:56,230 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,36193,1733063381174}] 2024-12-01T14:29:56,231 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf2 2024-12-01T14:29:56,231 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:56,231 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] regionserver.HRegion(1722): Closing 44c73b6c4a612bcfc6cf8a6997a1cd8e, disabling compactions & flushes 2024-12-01T14:29:56,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. after waiting 0 ms 2024-12-01T14:29:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] regionserver.HRegion(1676): Region close journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Waiting for close lock at 1733063396232Disabling compacts and flushes for region at 1733063396232Disabling writes for close at 1733063396232Writing region close event to WAL at 1733063396233 (+1 ms)Closed at 1733063396233 2024-12-01T14:29:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-01T14:29:56,389 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,389 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-01T14:29:56,390 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 44c73b6c4a612bcfc6cf8a6997a1cd8e, disabling compactions & flushes 2024-12-01T14:29:56,390 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,390 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,390 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. after waiting 0 ms 2024-12-01T14:29:56,390 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,390 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 44c73b6c4a612bcfc6cf8a6997a1cd8e 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-01T14:29:56,409 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf1/f0a01c305d6341ada7b866b75e7c1bdd is 35, key is r1/cf1:q/1733063396207/Put/seqid=0 2024-12-01T14:29:56,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741887_1065 (size=4783) 2024-12-01T14:29:56,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741887_1065 (size=4783) 2024-12-01T14:29:56,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741887_1065 (size=4783) 2024-12-01T14:29:56,416 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf1/f0a01c305d6341ada7b866b75e7c1bdd 2024-12-01T14:29:56,424 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf1/f0a01c305d6341ada7b866b75e7c1bdd as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/f0a01c305d6341ada7b866b75e7c1bdd 2024-12-01T14:29:56,430 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/f0a01c305d6341ada7b866b75e7c1bdd, entries=1, sequenceid=5, filesize=4.7 K 2024-12-01T14:29:56,432 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 44c73b6c4a612bcfc6cf8a6997a1cd8e in 42ms, sequenceid=5, compaction requested=false 2024-12-01T14:29:56,432 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-01T14:29:56,437 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-01T14:29:56,440 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,440 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Waiting for close lock at 1733063396390Running coprocessor pre-close hooks at 1733063396390Disabling compacts and flushes for region at 1733063396390Disabling writes for close at 1733063396390Obtaining lock to block concurrent updates at 1733063396390Preparing flush snapshotting stores in 44c73b6c4a612bcfc6cf8a6997a1cd8e at 1733063396390Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733063396391 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. at 1733063396392 (+1 ms)Flushing 44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1: creating writer at 1733063396392Flushing 44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1: appending metadata at 1733063396409 (+17 ms)Flushing 44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1: closing flushed file at 1733063396409Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66358aa6: reopening flushed file at 1733063396423 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 44c73b6c4a612bcfc6cf8a6997a1cd8e in 42ms, sequenceid=5, compaction requested=false at 1733063396432 (+9 ms)Writing region close event to WAL at 1733063396433 (+1 ms)Running coprocessor post-close hooks at 1733063396438 (+5 ms)Closed at 1733063396440 (+2 ms) 2024-12-01T14:29:56,440 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 44c73b6c4a612bcfc6cf8a6997a1cd8e move to 57b748500ef9,38129,1733063381264 record at close sequenceid=5 2024-12-01T14:29:56,444 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=CLOSED 2024-12-01T14:29:56,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,36193,1733063381174 because future has completed 2024-12-01T14:29:56,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-01T14:29:56,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,36193,1733063381174 in 219 msec 2024-12-01T14:29:56,455 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE; state=CLOSED, location=57b748500ef9,38129,1733063381264; forceNewPlan=false, retain=false 2024-12-01T14:29:56,605 INFO [57b748500ef9:34327 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T14:29:56,606 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=OPENING, regionLocation=57b748500ef9,38129,1733063381264 2024-12-01T14:29:56,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE because future has completed 2024-12-01T14:29:56,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,38129,1733063381264}] 2024-12-01T14:29:56,770 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,770 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 44c73b6c4a612bcfc6cf8a6997a1cd8e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:56,770 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,770 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:56,771 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,771 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,773 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,774 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf1 2024-12-01T14:29:56,774 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:56,783 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/f0a01c305d6341ada7b866b75e7c1bdd 2024-12-01T14:29:56,783 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:56,783 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,784 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf2 2024-12-01T14:29:56,784 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:56,785 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:56,785 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,786 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,787 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,788 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,788 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,789 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T14:29:56,790 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,791 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 44c73b6c4a612bcfc6cf8a6997a1cd8e; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65142982, jitterRate=-0.029293924570083618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T14:29:56,791 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:56,792 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Running coprocessor pre-open hook at 1733063396771Writing region info on filesystem at 1733063396771Initializing all the Stores at 1733063396772 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063396772Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063396773 (+1 ms)Cleaning up temporary data from old regions at 1733063396788 (+15 ms)Running coprocessor post-open hooks at 1733063396791 (+3 ms)Region opened successfully at 1733063396792 (+1 ms) 2024-12-01T14:29:56,794 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., pid=9, masterSystemTime=1733063396765 2024-12-01T14:29:56,796 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,796 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:56,797 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=OPEN, openSeqNum=9, regionLocation=57b748500ef9,38129,1733063381264 2024-12-01T14:29:56,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,38129,1733063381264 because future has completed 2024-12-01T14:29:56,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-01T14:29:56,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,38129,1733063381264 in 193 msec 2024-12-01T14:29:56,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE in 585 msec 2024-12-01T14:29:56,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T14:29:56,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T14:29:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:42124 deadline: 1733063456835, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=57b748500ef9 port=38129 startCode=1733063381264. As of locationSeqNum=5. 2024-12-01T14:29:56,864 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=57b748500ef9 port=38129 startCode=1733063381264. As of locationSeqNum=5. 2024-12-01T14:29:56,865 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=57b748500ef9 port=38129 startCode=1733063381264. As of locationSeqNum=5. 2024-12-01T14:29:56,865 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,38129,1733063381264, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=57b748500ef9 port=38129 startCode=1733063381264. As of locationSeqNum=5. 2024-12-01T14:29:56,987 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 44c73b6c4a612bcfc6cf8a6997a1cd8e 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-01T14:29:57,005 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf1/ddc0270605d245fda1dce49ab4074c0d is 29, key is r1/cf1:/1733063396977/DeleteFamily/seqid=0 2024-12-01T14:29:57,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741888_1066 (size=4906) 2024-12-01T14:29:57,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741888_1066 (size=4906) 2024-12-01T14:29:57,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741888_1066 (size=4906) 2024-12-01T14:29:57,013 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf1/ddc0270605d245fda1dce49ab4074c0d 2024-12-01T14:29:57,019 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ddc0270605d245fda1dce49ab4074c0d 2024-12-01T14:29:57,032 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf2/242b05f6b19643a0bab578c8176d757f is 29, key is r1/cf2:/1733063396977/DeleteFamily/seqid=0 2024-12-01T14:29:57,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741889_1067 (size=4906) 2024-12-01T14:29:57,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741889_1067 (size=4906) 2024-12-01T14:29:57,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741889_1067 (size=4906) 2024-12-01T14:29:57,040 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf2/242b05f6b19643a0bab578c8176d757f 2024-12-01T14:29:57,045 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 242b05f6b19643a0bab578c8176d757f 2024-12-01T14:29:57,046 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf1/ddc0270605d245fda1dce49ab4074c0d as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/ddc0270605d245fda1dce49ab4074c0d 2024-12-01T14:29:57,052 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ddc0270605d245fda1dce49ab4074c0d 2024-12-01T14:29:57,052 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/ddc0270605d245fda1dce49ab4074c0d, entries=1, sequenceid=12, filesize=4.8 K 2024-12-01T14:29:57,054 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf2/242b05f6b19643a0bab578c8176d757f as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/242b05f6b19643a0bab578c8176d757f 2024-12-01T14:29:57,060 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 242b05f6b19643a0bab578c8176d757f 2024-12-01T14:29:57,060 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/242b05f6b19643a0bab578c8176d757f, entries=1, sequenceid=12, filesize=4.8 K 2024-12-01T14:29:57,061 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 44c73b6c4a612bcfc6cf8a6997a1cd8e in 74ms, sequenceid=12, compaction requested=false 2024-12-01T14:29:57,061 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: 2024-12-01T14:29:57,063 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-01T14:29:57,065 DEBUG [Time-limited test {}] regionserver.HStore(1541): 44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1 is initiating major compaction (all files) 2024-12-01T14:29:57,065 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T14:29:57,065 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:57,066 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,066 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/f0a01c305d6341ada7b866b75e7c1bdd, hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/ddc0270605d245fda1dce49ab4074c0d] into tmpdir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp, totalSize=9.5 K 2024-12-01T14:29:57,067 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f0a01c305d6341ada7b866b75e7c1bdd, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733063396207 2024-12-01T14:29:57,068 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ddc0270605d245fda1dce49ab4074c0d, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-01T14:29:57,079 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 44c73b6c4a612bcfc6cf8a6997a1cd8e#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-01T14:29:57,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741890_1068 (size=4626) 2024-12-01T14:29:57,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741890_1068 (size=4626) 2024-12-01T14:29:57,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741890_1068 (size=4626) 2024-12-01T14:29:57,095 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf1/a984f32495e44e16a660534d9c5f7914 as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/a984f32495e44e16a660534d9c5f7914 2024-12-01T14:29:57,112 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1 of 44c73b6c4a612bcfc6cf8a6997a1cd8e into a984f32495e44e16a660534d9c5f7914(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T14:29:57,112 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: 2024-12-01T14:29:57,112 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-01T14:29:57,112 DEBUG [Time-limited test {}] regionserver.HStore(1541): 44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2 is initiating major compaction (all files) 2024-12-01T14:29:57,112 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T14:29:57,112 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T14:29:57,113 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,113 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/242b05f6b19643a0bab578c8176d757f] into tmpdir=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp, totalSize=4.8 K 2024-12-01T14:29:57,114 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 242b05f6b19643a0bab578c8176d757f, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-01T14:29:57,121 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 44c73b6c4a612bcfc6cf8a6997a1cd8e#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-01T14:29:57,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741891_1069 (size=4592) 2024-12-01T14:29:57,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741891_1069 (size=4592) 2024-12-01T14:29:57,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741891_1069 (size=4592) 2024-12-01T14:29:57,139 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/.tmp/cf2/196a0b164458436190abc6911406977f as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/196a0b164458436190abc6911406977f 2024-12-01T14:29:57,147 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2 of 44c73b6c4a612bcfc6cf8a6997a1cd8e into 196a0b164458436190abc6911406977f(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T14:29:57,147 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: 2024-12-01T14:29:57,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=44c73b6c4a612bcfc6cf8a6997a1cd8e, source=57b748500ef9,38129,1733063381264, destination=57b748500ef9,36193,1733063381174, warming up region on 57b748500ef9,36193,1733063381174 2024-12-01T14:29:57,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=44c73b6c4a612bcfc6cf8a6997a1cd8e, source=57b748500ef9,38129,1733063381264, destination=57b748500ef9,36193,1733063381174, running balancer 2024-12-01T14:29:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE 2024-12-01T14:29:57,154 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE 2024-12-01T14:29:57,155 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=CLOSING, regionLocation=57b748500ef9,38129,1733063381264 2024-12-01T14:29:57,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] regionserver.HRegion(7855): Warmup {ENCODED => 44c73b6c4a612bcfc6cf8a6997a1cd8e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:57,156 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,158 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf1 2024-12-01T14:29:57,158 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:57,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE because future has completed 2024-12-01T14:29:57,159 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-01T14:29:57,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,38129,1733063381264}] 2024-12-01T14:29:57,167 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/a984f32495e44e16a660534d9c5f7914 2024-12-01T14:29:57,172 INFO [StoreFileOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ddc0270605d245fda1dce49ab4074c0d 2024-12-01T14:29:57,172 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/ddc0270605d245fda1dce49ab4074c0d 2024-12-01T14:29:57,178 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/f0a01c305d6341ada7b866b75e7c1bdd 2024-12-01T14:29:57,179 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:57,179 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,180 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf2 2024-12-01T14:29:57,180 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:57,186 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/196a0b164458436190abc6911406977f 2024-12-01T14:29:57,192 INFO [StoreFileOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 242b05f6b19643a0bab578c8176d757f 2024-12-01T14:29:57,192 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/242b05f6b19643a0bab578c8176d757f 2024-12-01T14:29:57,192 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] regionserver.HRegion(1722): Closing 44c73b6c4a612bcfc6cf8a6997a1cd8e, disabling compactions & flushes 2024-12-01T14:29:57,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. after waiting 0 ms 2024-12-01T14:29:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36193 {}] regionserver.HRegion(1676): Region close journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Waiting for close lock at 1733063397192Disabling compacts and flushes for region at 1733063397192Disabling writes for close at 1733063397193 (+1 ms)Writing region close event to WAL at 1733063397195 (+2 ms)Closed at 1733063397195 2024-12-01T14:29:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-01T14:29:57,314 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,314 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-01T14:29:57,314 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 44c73b6c4a612bcfc6cf8a6997a1cd8e, disabling compactions & flushes 2024-12-01T14:29:57,314 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,314 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,314 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. after waiting 0 ms 2024-12-01T14:29:57,314 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,315 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/f0a01c305d6341ada7b866b75e7c1bdd, hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/ddc0270605d245fda1dce49ab4074c0d] to archive 2024-12-01T14:29:57,317 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T14:29:57,320 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/f0a01c305d6341ada7b866b75e7c1bdd to hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/f0a01c305d6341ada7b866b75e7c1bdd 2024-12-01T14:29:57,322 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/ddc0270605d245fda1dce49ab4074c0d to hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/ddc0270605d245fda1dce49ab4074c0d 2024-12-01T14:29:57,334 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/242b05f6b19643a0bab578c8176d757f] to archive 2024-12-01T14:29:57,335 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T14:29:57,338 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/242b05f6b19643a0bab578c8176d757f to hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/242b05f6b19643a0bab578c8176d757f 2024-12-01T14:29:57,343 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-01T14:29:57,344 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,344 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Waiting for close lock at 1733063397314Running coprocessor pre-close hooks at 1733063397314Disabling compacts and flushes for region at 1733063397314Disabling writes for close at 1733063397314Writing region close event to WAL at 1733063397339 (+25 ms)Running coprocessor post-close hooks at 1733063397344 (+5 ms)Closed at 1733063397344 2024-12-01T14:29:57,344 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 44c73b6c4a612bcfc6cf8a6997a1cd8e move to 57b748500ef9,36193,1733063381174 record at close sequenceid=12 2024-12-01T14:29:57,347 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,348 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=CLOSED 2024-12-01T14:29:57,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,38129,1733063381264 because future has completed 2024-12-01T14:29:57,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-01T14:29:57,355 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,38129,1733063381264 in 193 msec 2024-12-01T14:29:57,356 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE; state=CLOSED, location=57b748500ef9,36193,1733063381174; forceNewPlan=false, retain=false 2024-12-01T14:29:57,506 INFO [57b748500ef9:34327 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-01T14:29:57,506 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=OPENING, regionLocation=57b748500ef9,36193,1733063381174 2024-12-01T14:29:57,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE because future has completed 2024-12-01T14:29:57,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,36193,1733063381174}] 2024-12-01T14:29:57,671 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,671 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 44c73b6c4a612bcfc6cf8a6997a1cd8e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:57,672 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,672 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:57,672 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,672 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,674 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,676 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf1 2024-12-01T14:29:57,676 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:57,684 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/a984f32495e44e16a660534d9c5f7914 2024-12-01T14:29:57,684 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:57,684 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,685 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf2 2024-12-01T14:29:57,685 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:57,691 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/196a0b164458436190abc6911406977f 2024-12-01T14:29:57,692 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:57,692 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,693 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,694 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,695 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,695 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,696 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T14:29:57,697 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,698 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 44c73b6c4a612bcfc6cf8a6997a1cd8e; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68171697, jitterRate=0.015837445855140686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T14:29:57,698 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,698 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Running coprocessor pre-open hook at 1733063397673Writing region info on filesystem at 1733063397673Initializing all the Stores at 1733063397674 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063397674Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063397674Cleaning up temporary data from old regions at 1733063397695 (+21 ms)Running coprocessor post-open hooks at 1733063397698 (+3 ms)Region opened successfully at 1733063397698 2024-12-01T14:29:57,699 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., pid=12, masterSystemTime=1733063397663 2024-12-01T14:29:57,702 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,702 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,702 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=OPEN, openSeqNum=18, regionLocation=57b748500ef9,36193,1733063381174 2024-12-01T14:29:57,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,36193,1733063381174 because future has completed 2024-12-01T14:29:57,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-01T14:29:57,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,36193,1733063381174 in 197 msec 2024-12-01T14:29:57,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, REOPEN/MOVE in 557 msec 2024-12-01T14:29:57,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T14:29:57,758 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T14:29:57,759 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 57b748500ef9,36193,1733063381174: testing ***** 2024-12-01T14:29:57,760 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-01T14:29:57,762 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-01T14:29:57,763 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-01T14:29:57,765 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-01T14:29:57,765 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-01T14:29:57,774 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 326750296 }, "NonHeapMemoryUsage": { "committed": 171376640, "init": 7667712, "max": -1, "used": 168712336 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "57b748500ef9", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "57b748500ef9", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56293", "tag.serverName": "57b748500ef9,40449,1733063381307", "tag.clusterId": "13e07f3e-13f5-4a4b-89f8-14b649bbc29b", "tag.Context": "regionserver", "tag.Hostname": "57b748500ef9", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733063381307, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-01T14:29:57,778 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34327 {}] master.MasterRpcServices(700): 57b748500ef9,36193,1733063381174 reported a fatal error: ***** ABORTING region server 57b748500ef9,36193,1733063381174: testing ***** 2024-12-01T14:29:57,780 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '57b748500ef9,36193,1733063381174' ***** 2024-12-01T14:29:57,781 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-01T14:29:57,781 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T14:29:57,781 INFO [RS:0;57b748500ef9:36193 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-01T14:29:57,781 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T14:29:57,781 INFO [RS:0;57b748500ef9:36193 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-01T14:29:57,781 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(3091): Received CLOSE for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,782 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(956): aborting server 57b748500ef9,36193,1733063381174 2024-12-01T14:29:57,782 INFO [RS:0;57b748500ef9:36193 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:29:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38129 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:47842 deadline: 1733063457782, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=57b748500ef9 port=36193 startCode=1733063381174. As of locationSeqNum=12. 2024-12-01T14:29:57,782 INFO [RS:0;57b748500ef9:36193 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;57b748500ef9:36193. 2024-12-01T14:29:57,782 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 44c73b6c4a612bcfc6cf8a6997a1cd8e, disabling compactions & flushes 2024-12-01T14:29:57,782 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,783 DEBUG [RS:0;57b748500ef9:36193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:29:57,783 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,783 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. after waiting 0 ms 2024-12-01T14:29:57,783 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,783 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,38129,1733063381264, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,38129,1733063381264, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=57b748500ef9 port=36193 startCode=1733063381174. As of locationSeqNum=12. 2024-12-01T14:29:57,783 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,38129,1733063381264, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=57b748500ef9 port=36193 startCode=1733063381174. As of locationSeqNum=12. 2024-12-01T14:29:57,783 DEBUG [RS:0;57b748500ef9:36193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:29:57,783 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,38129,1733063381264, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=57b748500ef9 port=36193 startCode=1733063381174. As of locationSeqNum=12. 2024-12-01T14:29:57,783 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-01T14:29:57,783 DEBUG [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(1325): Online Regions={44c73b6c4a612bcfc6cf8a6997a1cd8e=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.} 2024-12-01T14:29:57,784 DEBUG [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(1351): Waiting on 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:57,786 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,786 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Waiting for close lock at 1733063397782Running coprocessor pre-close hooks at 1733063397782Disabling compacts and flushes for region at 1733063397782Disabling writes for close at 1733063397783 (+1 ms)Writing region close event to WAL at 1733063397785 (+2 ms)Running coprocessor post-close hooks at 1733063397785Closed at 1733063397785 2024-12-01T14:29:57,786 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:57,843 INFO [regionserver/57b748500ef9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:29:57,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 57b748500ef9,36193,1733063381174 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:29:57,898 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 57b748500ef9,36193,1733063381174 aborting 2024-12-01T14:29:57,898 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 57b748500ef9,36193,1733063381174 aborting 2024-12-01T14:29:57,898 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=12 from cache 2024-12-01T14:29:57,984 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(976): stopping server 57b748500ef9,36193,1733063381174; all regions closed. 2024-12-01T14:29:57,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741834_1010 (size=1406) 2024-12-01T14:29:57,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741834_1010 (size=1406) 2024-12-01T14:29:57,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741834_1010 (size=1406) 2024-12-01T14:29:57,989 DEBUG [RS:0;57b748500ef9:36193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:29:57,989 INFO [RS:0;57b748500ef9:36193 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:29:57,989 INFO [RS:0;57b748500ef9:36193 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:29:57,990 INFO [RS:0;57b748500ef9:36193 {}] hbase.ChoreService(370): Chore service for: regionserver/57b748500ef9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T14:29:57,990 INFO [regionserver/57b748500ef9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:29:57,990 INFO [RS:0;57b748500ef9:36193 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T14:29:57,990 INFO [RS:0;57b748500ef9:36193 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T14:29:57,990 INFO [RS:0;57b748500ef9:36193 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T14:29:57,990 INFO [RS:0;57b748500ef9:36193 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:29:57,991 INFO [RS:0;57b748500ef9:36193 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36193 2024-12-01T14:29:58,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:29:58,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/57b748500ef9,36193,1733063381174 2024-12-01T14:29:58,074 INFO [RS:0;57b748500ef9:36193 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:29:58,077 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [57b748500ef9,36193,1733063381174] 2024-12-01T14:29:58,100 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/57b748500ef9,36193,1733063381174 already deleted, retry=false 2024-12-01T14:29:58,101 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 57b748500ef9,36193,1733063381174 on 57b748500ef9,34327,1733063380396 2024-12-01T14:29:58,105 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 57b748500ef9,36193,1733063381174, splitWal=true, meta=false 2024-12-01T14:29:58,107 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 57b748500ef9,36193,1733063381174 (carryingMeta=false) 57b748500ef9,36193,1733063381174/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@5b301a83[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-01T14:29:58,108 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 57b748500ef9,36193,1733063381174, splitWal=true, meta=false 2024-12-01T14:29:58,110 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 57b748500ef9,36193,1733063381174 had 1 regions 2024-12-01T14:29:58,110 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=18] 2024-12-01T14:29:58,111 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 57b748500ef9,36193,1733063381174, splitWal=true, meta=false, isMeta: false 2024-12-01T14:29:58,111 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 57b748500ef9:36193 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 57b748500ef9/172.17.0.3:36193 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:29:58,112 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=18, error=java.net.ConnectException: Call to address=57b748500ef9:36193 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 57b748500ef9/172.17.0.3:36193 2024-12-01T14:29:58,113 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting 2024-12-01T14:29:58,113 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-01T14:29:58,113 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=18 from cache 2024-12-01T14:29:58,113 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 57b748500ef9:36193 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 57b748500ef9/172.17.0.3:36193 2024-12-01T14:29:58,114 INFO [PEWorker-1 {}] master.SplitWALManager(105): 57b748500ef9,36193,1733063381174 WAL count=1, meta=false 2024-12-01T14:29:58,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 57b748500ef9%2C36193%2C1733063381174.1733063382781}] 2024-12-01T14:29:58,121 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=57b748500ef9,38129,1733063381264 2024-12-01T14:29:58,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 57b748500ef9%2C36193%2C1733063381174.1733063382781, worker=57b748500ef9,38129,1733063381264}] 2024-12-01T14:29:58,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:29:58,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36193-0x101914c865a0001, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:29:58,191 INFO [RS:0;57b748500ef9:36193 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:29:58,191 INFO [RS:0;57b748500ef9:36193 {}] regionserver.HRegionServer(1031): Exiting; stopping=57b748500ef9,36193,1733063381174; zookeeper connection closed. 2024-12-01T14:29:58,191 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@29592262 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@29592262 2024-12-01T14:29:58,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38129 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-01T14:29:58,299 INFO [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781, size=1.4 K (1406bytes) 2024-12-01T14:29:58,299 INFO [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781 2024-12-01T14:29:58,300 INFO [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781 after 1ms 2024-12-01T14:29:58,302 DEBUG [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:58,302 INFO [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781 took 3ms 2024-12-01T14:29:58,309 DEBUG [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 44c73b6c4a612bcfc6cf8a6997a1cd8e: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-01T14:29:58,309 DEBUG [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781 so closing down 2024-12-01T14:29:58,309 DEBUG [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:58,310 INFO [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:58,310 INFO [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781, size=1.4 K, length=1406, corrupted=false, cancelled=false 2024-12-01T14:29:58,310 DEBUG [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781, journal: Splitting hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781, size=1.4 K (1406bytes) at 1733063398299Finishing writing output for hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781 so closing down at 1733063398309 (+10 ms)3 split writer threads finished at 1733063398310 (+1 ms)Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781, size=1.4 K, length=1406, corrupted=false, cancelled=false at 1733063398310 2024-12-01T14:29:58,310 DEBUG [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781 2024-12-01T14:29:58,311 DEBUG [RS_LOG_REPLAY_OPS-regionserver/57b748500ef9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-01T14:29:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34327 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-01T14:29:58,318 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting/57b748500ef9%2C36193%2C1733063381174.1733063382781 to hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/oldWALs 2024-12-01T14:29:58,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-01T14:29:58,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 57b748500ef9%2C36193%2C1733063381174.1733063382781, worker=57b748500ef9,38129,1733063381264 in 195 msec 2024-12-01T14:29:58,322 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=57b748500ef9,38129,1733063381264 2024-12-01T14:29:58,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-01T14:29:58,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 57b748500ef9%2C36193%2C1733063381174.1733063382781, worker=57b748500ef9,38129,1733063381264 in 207 msec 2024-12-01T14:29:58,326 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting dir is empty, no logs to split. 2024-12-01T14:29:58,327 INFO [PEWorker-1 {}] master.SplitWALManager(105): 57b748500ef9,36193,1733063381174 WAL count=0, meta=false 2024-12-01T14:29:58,327 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 57b748500ef9,36193,1733063381174 WAL splitting is done? wals=0, meta=false 2024-12-01T14:29:58,329 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 57b748500ef9,36193,1733063381174 failed, ignore...File hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,36193,1733063381174-splitting does not exist. 2024-12-01T14:29:58,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, ASSIGN}] 2024-12-01T14:29:58,333 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, ASSIGN 2024-12-01T14:29:58,334 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-01T14:29:58,419 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=18] 2024-12-01T14:29:58,420 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 57b748500ef9:36193 this server is in the failed servers list 2024-12-01T14:29:58,420 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=57b748500ef9:36193 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 57b748500ef9:36193 2024-12-01T14:29:58,420 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 57b748500ef9:36193 2024-12-01T14:29:58,420 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,36193,1733063381174, seqNum=18 from cache 2024-12-01T14:29:58,484 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(204): Hosts are {57b748500ef9=0} racks are {/default-rack=0} 2024-12-01T14:29:58,485 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-01T14:29:58,485 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-01T14:29:58,485 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-01T14:29:58,485 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-01T14:29:58,485 INFO [57b748500ef9:34327 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-01T14:29:58,485 INFO [57b748500ef9:34327 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-01T14:29:58,485 DEBUG [57b748500ef9:34327 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-01T14:29:58,485 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=OPENING, regionLocation=57b748500ef9,40449,1733063381307 2024-12-01T14:29:58,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, ASSIGN because future has completed 2024-12-01T14:29:58,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,40449,1733063381307}] 2024-12-01T14:29:58,644 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T14:29:58,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35905, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T14:29:58,658 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:58,658 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 44c73b6c4a612bcfc6cf8a6997a1cd8e, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:58,659 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,659 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:58,659 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,659 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,661 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,662 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf1 2024-12-01T14:29:58,662 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:58,669 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1/a984f32495e44e16a660534d9c5f7914 2024-12-01T14:29:58,669 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:58,669 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,670 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44c73b6c4a612bcfc6cf8a6997a1cd8e columnFamilyName cf2 2024-12-01T14:29:58,670 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:58,677 DEBUG [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2/196a0b164458436190abc6911406977f 2024-12-01T14:29:58,677 INFO [StoreOpener-44c73b6c4a612bcfc6cf8a6997a1cd8e-1 {}] regionserver.HStore(327): Store=44c73b6c4a612bcfc6cf8a6997a1cd8e/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:58,677 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,679 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,680 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,681 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,681 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,682 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-01T14:29:58,684 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,685 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 44c73b6c4a612bcfc6cf8a6997a1cd8e; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70935214, jitterRate=0.05701705813407898}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-01T14:29:58,685 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:29:58,686 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Running coprocessor pre-open hook at 1733063398659Writing region info on filesystem at 1733063398659Initializing all the Stores at 1733063398661 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063398661Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063398661Cleaning up temporary data from old regions at 1733063398681 (+20 ms)Running coprocessor post-open hooks at 1733063398685 (+4 ms)Region opened successfully at 1733063398686 (+1 ms) 2024-12-01T14:29:58,687 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., pid=17, masterSystemTime=1733063398644 2024-12-01T14:29:58,690 DEBUG [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:58,690 INFO [RS_OPEN_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:29:58,690 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=44c73b6c4a612bcfc6cf8a6997a1cd8e, regionState=OPEN, openSeqNum=18, regionLocation=57b748500ef9,40449,1733063381307 2024-12-01T14:29:58,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,40449,1733063381307 because future has completed 2024-12-01T14:29:58,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-01T14:29:58,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 44c73b6c4a612bcfc6cf8a6997a1cd8e, server=57b748500ef9,40449,1733063381307 in 202 msec 2024-12-01T14:29:58,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-01T14:29:58,699 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 57b748500ef9,36193,1733063381174 after splitting done 2024-12-01T14:29:58,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=44c73b6c4a612bcfc6cf8a6997a1cd8e, ASSIGN in 365 msec 2024-12-01T14:29:58,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 57b748500ef9,36193,1733063381174, splitWal=true, meta=false in 598 msec 2024-12-01T14:29:58,941 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e., hostname=57b748500ef9,40449,1733063381307, seqNum=18] 2024-12-01T14:29:58,941 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T14:29:58,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T14:29:58,966 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=415 (was 420), OpenFileDescriptor=1051 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=200 (was 200), ProcessCount=11 (was 11), AvailableMemoryMB=4925 (was 4953) 2024-12-01T14:29:58,966 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-01T14:29:58,981 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=415, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=200, ProcessCount=11, AvailableMemoryMB=4925 2024-12-01T14:29:58,981 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-01T14:29:58,999 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:59,002 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:59,003 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:59,005 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36327149, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/hregion-36327149, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:59,018 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36327149/hregion-36327149.1733063399006, exclude list is [], retry=0 2024-12-01T14:29:59,020 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:59,020 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:59,021 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:59,022 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36327149/hregion-36327149.1733063399006 2024-12-01T14:29:59,023 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:29:59,023 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ba340309b8d5e8cf235ac96fc4af0cd3, NAME => 'testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/hbase 2024-12-01T14:29:59,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741893_1071 (size=67) 2024-12-01T14:29:59,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741893_1071 (size=67) 2024-12-01T14:29:59,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741893_1071 (size=67) 2024-12-01T14:29:59,035 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:59,037 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,038 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba340309b8d5e8cf235ac96fc4af0cd3 columnFamilyName a 2024-12-01T14:29:59,038 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,039 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(327): Store=ba340309b8d5e8cf235ac96fc4af0cd3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,039 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,040 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba340309b8d5e8cf235ac96fc4af0cd3 columnFamilyName b 2024-12-01T14:29:59,040 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,041 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(327): Store=ba340309b8d5e8cf235ac96fc4af0cd3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,041 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,042 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba340309b8d5e8cf235ac96fc4af0cd3 columnFamilyName c 2024-12-01T14:29:59,043 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,043 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(327): Store=ba340309b8d5e8cf235ac96fc4af0cd3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,043 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,044 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,045 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,046 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,046 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,047 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:59,048 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,051 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:59,051 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ba340309b8d5e8cf235ac96fc4af0cd3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59379033, jitterRate=-0.11518345773220062}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:59,052 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ba340309b8d5e8cf235ac96fc4af0cd3: Writing region info on filesystem at 1733063399035Initializing all the Stores at 1733063399036 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399036Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399036Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399037 (+1 ms)Cleaning up temporary data from old regions at 1733063399046 (+9 ms)Region opened successfully at 1733063399052 (+6 ms) 2024-12-01T14:29:59,052 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ba340309b8d5e8cf235ac96fc4af0cd3, disabling compactions & flushes 2024-12-01T14:29:59,052 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. 2024-12-01T14:29:59,052 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. 2024-12-01T14:29:59,053 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. after waiting 0 ms 2024-12-01T14:29:59,053 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. 2024-12-01T14:29:59,053 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. 2024-12-01T14:29:59,053 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ba340309b8d5e8cf235ac96fc4af0cd3: Waiting for close lock at 1733063399052Disabling compacts and flushes for region at 1733063399052Disabling writes for close at 1733063399053 (+1 ms)Writing region close event to WAL at 1733063399053Closed at 1733063399053 2024-12-01T14:29:59,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741892_1070 (size=95) 2024-12-01T14:29:59,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741892_1070 (size=95) 2024-12-01T14:29:59,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741892_1070 (size=95) 2024-12-01T14:29:59,062 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:29:59,062 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36327149:(num 1733063399006) 2024-12-01T14:29:59,062 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:59,064 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:59,082 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065, exclude list is [], retry=0 2024-12-01T14:29:59,085 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:59,086 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:59,086 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:59,088 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065 2024-12-01T14:29:59,088 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289)] 2024-12-01T14:29:59,088 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ba340309b8d5e8cf235ac96fc4af0cd3, NAME => 'testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:59,089 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:59,089 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,089 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,090 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,091 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba340309b8d5e8cf235ac96fc4af0cd3 columnFamilyName a 2024-12-01T14:29:59,091 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,092 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(327): Store=ba340309b8d5e8cf235ac96fc4af0cd3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,092 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,093 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba340309b8d5e8cf235ac96fc4af0cd3 columnFamilyName b 2024-12-01T14:29:59,093 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,093 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(327): Store=ba340309b8d5e8cf235ac96fc4af0cd3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,093 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,094 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba340309b8d5e8cf235ac96fc4af0cd3 columnFamilyName c 2024-12-01T14:29:59,094 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,095 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(327): Store=ba340309b8d5e8cf235ac96fc4af0cd3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,095 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,096 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,097 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,099 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,099 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,099 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:59,101 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,102 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ba340309b8d5e8cf235ac96fc4af0cd3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61466820, jitterRate=-0.084073007106781}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:59,103 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ba340309b8d5e8cf235ac96fc4af0cd3: Writing region info on filesystem at 1733063399089Initializing all the Stores at 1733063399090 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399090Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399090Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399090Cleaning up temporary data from old regions at 1733063399099 (+9 ms)Region opened successfully at 1733063399102 (+3 ms) 2024-12-01T14:29:59,132 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ba340309b8d5e8cf235ac96fc4af0cd3 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-01T14:29:59,154 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/a/83ae3050a4a6455f96adf06ba9550d4a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733063399103/Put/seqid=0 2024-12-01T14:29:59,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741895_1073 (size=5958) 2024-12-01T14:29:59,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741895_1073 (size=5958) 2024-12-01T14:29:59,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741895_1073 (size=5958) 2024-12-01T14:29:59,163 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/a/83ae3050a4a6455f96adf06ba9550d4a 2024-12-01T14:29:59,190 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/b/c7e924bd937f49979d501e6b7a0f05a2 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733063399112/Put/seqid=0 2024-12-01T14:29:59,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741896_1074 (size=5958) 2024-12-01T14:29:59,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741896_1074 (size=5958) 2024-12-01T14:29:59,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741896_1074 (size=5958) 2024-12-01T14:29:59,199 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/b/c7e924bd937f49979d501e6b7a0f05a2 2024-12-01T14:29:59,220 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/c/8065bdc1a25a4ffb8676eac9e74d9f4e is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733063399121/Put/seqid=0 2024-12-01T14:29:59,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741897_1075 (size=5958) 2024-12-01T14:29:59,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741897_1075 (size=5958) 2024-12-01T14:29:59,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741897_1075 (size=5958) 2024-12-01T14:29:59,229 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/c/8065bdc1a25a4ffb8676eac9e74d9f4e 2024-12-01T14:29:59,236 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/a/83ae3050a4a6455f96adf06ba9550d4a as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/a/83ae3050a4a6455f96adf06ba9550d4a 2024-12-01T14:29:59,243 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/a/83ae3050a4a6455f96adf06ba9550d4a, entries=10, sequenceid=33, filesize=5.8 K 2024-12-01T14:29:59,244 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/b/c7e924bd937f49979d501e6b7a0f05a2 as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/b/c7e924bd937f49979d501e6b7a0f05a2 2024-12-01T14:29:59,251 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/b/c7e924bd937f49979d501e6b7a0f05a2, entries=10, sequenceid=33, filesize=5.8 K 2024-12-01T14:29:59,253 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/c/8065bdc1a25a4ffb8676eac9e74d9f4e as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/c/8065bdc1a25a4ffb8676eac9e74d9f4e 2024-12-01T14:29:59,260 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/c/8065bdc1a25a4ffb8676eac9e74d9f4e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-01T14:29:59,261 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for ba340309b8d5e8cf235ac96fc4af0cd3 in 129ms, sequenceid=33, compaction requested=false 2024-12-01T14:29:59,261 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ba340309b8d5e8cf235ac96fc4af0cd3: 2024-12-01T14:29:59,261 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ba340309b8d5e8cf235ac96fc4af0cd3, disabling compactions & flushes 2024-12-01T14:29:59,261 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. 2024-12-01T14:29:59,261 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. 2024-12-01T14:29:59,261 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. after waiting 0 ms 2024-12-01T14:29:59,262 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. 2024-12-01T14:29:59,265 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. 2024-12-01T14:29:59,265 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ba340309b8d5e8cf235ac96fc4af0cd3: Waiting for close lock at 1733063399261Disabling compacts and flushes for region at 1733063399261Disabling writes for close at 1733063399261Writing region close event to WAL at 1733063399265 (+4 ms)Closed at 1733063399265 2024-12-01T14:29:59,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741894_1072 (size=3384) 2024-12-01T14:29:59,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741894_1072 (size=3384) 2024-12-01T14:29:59,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741894_1072 (size=3384) 2024-12-01T14:29:59,273 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/b/c7e924bd937f49979d501e6b7a0f05a2 to hdfs://localhost:40889/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/b/c7e924bd937f49979d501e6b7a0f05a2 2024-12-01T14:29:59,289 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065, size=3.3 K (3384bytes) 2024-12-01T14:29:59,290 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065 2024-12-01T14:29:59,290 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065 after 0ms 2024-12-01T14:29:59,293 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:59,293 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065 took 4ms 2024-12-01T14:29:59,296 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065 so closing down 2024-12-01T14:29:59,296 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:29:59,296 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733063399065.temp 2024-12-01T14:29:59,298 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000003-wal.1733063399065.temp 2024-12-01T14:29:59,298 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:29:59,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741898_1076 (size=2944) 2024-12-01T14:29:59,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741898_1076 (size=2944) 2024-12-01T14:29:59,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741898_1076 (size=2944) 2024-12-01T14:29:59,307 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000003-wal.1733063399065.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-01T14:29:59,308 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000003-wal.1733063399065.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000032 2024-12-01T14:29:59,308 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065, size=3.3 K, length=3384, corrupted=false, cancelled=false 2024-12-01T14:29:59,308 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065, journal: Splitting hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065, size=3.3 K (3384bytes) at 1733063399290Finishing writing output for hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065 so closing down at 1733063399296 (+6 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000003-wal.1733063399065.temp at 1733063399298 (+2 ms)3 split writer threads finished at 1733063399298Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000003-wal.1733063399065.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733063399307 (+9 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000003-wal.1733063399065.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000032 at 1733063399308 (+1 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065, size=3.3 K, length=3384, corrupted=false, cancelled=false at 1733063399308 2024-12-01T14:29:59,310 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399065 to hdfs://localhost:40889/hbase/oldWALs/wal.1733063399065 2024-12-01T14:29:59,311 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000032 2024-12-01T14:29:59,312 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:59,314 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:59,326 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399314, exclude list is [], retry=0 2024-12-01T14:29:59,329 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:59,329 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:59,329 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:59,331 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399314 2024-12-01T14:29:59,331 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:29:59,332 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ba340309b8d5e8cf235ac96fc4af0cd3, NAME => 'testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:59,332 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:59,332 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,332 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,334 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,335 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba340309b8d5e8cf235ac96fc4af0cd3 columnFamilyName a 2024-12-01T14:29:59,335 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,341 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/a/83ae3050a4a6455f96adf06ba9550d4a 2024-12-01T14:29:59,341 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(327): Store=ba340309b8d5e8cf235ac96fc4af0cd3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,341 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,343 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba340309b8d5e8cf235ac96fc4af0cd3 columnFamilyName b 2024-12-01T14:29:59,343 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,344 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(327): Store=ba340309b8d5e8cf235ac96fc4af0cd3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,344 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,345 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba340309b8d5e8cf235ac96fc4af0cd3 columnFamilyName c 2024-12-01T14:29:59,345 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,350 DEBUG [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/c/8065bdc1a25a4ffb8676eac9e74d9f4e 2024-12-01T14:29:59,351 INFO [StoreOpener-ba340309b8d5e8cf235ac96fc4af0cd3-1 {}] regionserver.HStore(327): Store=ba340309b8d5e8cf235ac96fc4af0cd3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,351 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,351 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,353 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,353 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000032 2024-12-01T14:29:59,355 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:29:59,357 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000032 2024-12-01T14:29:59,357 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ba340309b8d5e8cf235ac96fc4af0cd3 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-01T14:29:59,371 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/b/f5a67f5a0f6941f183ee38ae4f23fab0 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733063399112/Put/seqid=0 2024-12-01T14:29:59,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741900_1078 (size=5958) 2024-12-01T14:29:59,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741900_1078 (size=5958) 2024-12-01T14:29:59,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741900_1078 (size=5958) 2024-12-01T14:29:59,380 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/b/f5a67f5a0f6941f183ee38ae4f23fab0 2024-12-01T14:29:59,385 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/.tmp/b/f5a67f5a0f6941f183ee38ae4f23fab0 as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/b/f5a67f5a0f6941f183ee38ae4f23fab0 2024-12-01T14:29:59,391 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/b/f5a67f5a0f6941f183ee38ae4f23fab0, entries=10, sequenceid=32, filesize=5.8 K 2024-12-01T14:29:59,391 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ba340309b8d5e8cf235ac96fc4af0cd3 in 34ms, sequenceid=32, compaction requested=false; wal=null 2024-12-01T14:29:59,392 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/0000000000000000032 2024-12-01T14:29:59,393 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,393 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,394 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:59,395 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ba340309b8d5e8cf235ac96fc4af0cd3 2024-12-01T14:29:59,397 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/ba340309b8d5e8cf235ac96fc4af0cd3/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-01T14:29:59,398 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ba340309b8d5e8cf235ac96fc4af0cd3; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61872120, jitterRate=-0.07803356647491455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:59,399 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ba340309b8d5e8cf235ac96fc4af0cd3: Writing region info on filesystem at 1733063399332Initializing all the Stores at 1733063399333 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399333Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399334 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399334Obtaining lock to block concurrent updates at 1733063399357 (+23 ms)Preparing flush snapshotting stores in ba340309b8d5e8cf235ac96fc4af0cd3 at 1733063399357Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733063399357Flushing stores of testReplayEditsWrittenViaHRegion,,1733063399000.ba340309b8d5e8cf235ac96fc4af0cd3. at 1733063399357Flushing ba340309b8d5e8cf235ac96fc4af0cd3/b: creating writer at 1733063399357Flushing ba340309b8d5e8cf235ac96fc4af0cd3/b: appending metadata at 1733063399370 (+13 ms)Flushing ba340309b8d5e8cf235ac96fc4af0cd3/b: closing flushed file at 1733063399371 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1aa69a1f: reopening flushed file at 1733063399384 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for ba340309b8d5e8cf235ac96fc4af0cd3 in 34ms, sequenceid=32, compaction requested=false; wal=null at 1733063399391 (+7 ms)Cleaning up temporary data from old regions at 1733063399393 (+2 ms)Region opened successfully at 1733063399399 (+6 ms) 2024-12-01T14:29:59,419 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=424 (was 415) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:56112 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:55938 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:54246 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:41016 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:54116 [Waiting for operation #41] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:41172 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=200 (was 200), ProcessCount=11 (was 11), AvailableMemoryMB=4920 (was 4925) 2024-12-01T14:29:59,420 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-12-01T14:29:59,433 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=424, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=200, ProcessCount=11, AvailableMemoryMB=4920 2024-12-01T14:29:59,433 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-12-01T14:29:59,451 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:59,454 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:29:59,454 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:29:59,457 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-32647534, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/hregion-32647534, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:59,468 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-32647534/hregion-32647534.1733063399457, exclude list is [], retry=0 2024-12-01T14:29:59,471 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:59,472 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:59,472 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:59,475 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-32647534/hregion-32647534.1733063399457 2024-12-01T14:29:59,476 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:40289:40289)] 2024-12-01T14:29:59,476 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e9390d8c0de9aae718ab828acfcab4f9, NAME => 'testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/hbase 2024-12-01T14:29:59,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741902_1080 (size=68) 2024-12-01T14:29:59,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741902_1080 (size=68) 2024-12-01T14:29:59,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741902_1080 (size=68) 2024-12-01T14:29:59,484 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:59,486 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,487 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9390d8c0de9aae718ab828acfcab4f9 columnFamilyName a 2024-12-01T14:29:59,487 DEBUG [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,487 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(327): Store=e9390d8c0de9aae718ab828acfcab4f9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,487 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,489 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9390d8c0de9aae718ab828acfcab4f9 columnFamilyName b 2024-12-01T14:29:59,489 DEBUG [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,489 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(327): Store=e9390d8c0de9aae718ab828acfcab4f9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,489 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,490 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9390d8c0de9aae718ab828acfcab4f9 columnFamilyName c 2024-12-01T14:29:59,490 DEBUG [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,491 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(327): Store=e9390d8c0de9aae718ab828acfcab4f9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,491 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,492 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,492 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,493 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,493 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,494 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:59,495 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,498 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:29:59,498 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9390d8c0de9aae718ab828acfcab4f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64958230, jitterRate=-0.03204694390296936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:59,499 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9390d8c0de9aae718ab828acfcab4f9: Writing region info on filesystem at 1733063399484Initializing all the Stores at 1733063399485 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399485Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399485Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399485Cleaning up temporary data from old regions at 1733063399493 (+8 ms)Region opened successfully at 1733063399499 (+6 ms) 2024-12-01T14:29:59,500 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e9390d8c0de9aae718ab828acfcab4f9, disabling compactions & flushes 2024-12-01T14:29:59,500 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. 2024-12-01T14:29:59,500 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. 2024-12-01T14:29:59,500 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. after waiting 0 ms 2024-12-01T14:29:59,500 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. 2024-12-01T14:29:59,500 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. 2024-12-01T14:29:59,500 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e9390d8c0de9aae718ab828acfcab4f9: Waiting for close lock at 1733063399499Disabling compacts and flushes for region at 1733063399499Disabling writes for close at 1733063399500 (+1 ms)Writing region close event to WAL at 1733063399500Closed at 1733063399500 2024-12-01T14:29:59,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741901_1079 (size=95) 2024-12-01T14:29:59,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741901_1079 (size=95) 2024-12-01T14:29:59,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741901_1079 (size=95) 2024-12-01T14:29:59,505 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:29:59,505 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-32647534:(num 1733063399457) 2024-12-01T14:29:59,505 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:29:59,507 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:29:59,520 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508, exclude list is [], retry=0 2024-12-01T14:29:59,522 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:29:59,523 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:29:59,523 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:29:59,526 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508 2024-12-01T14:29:59,526 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:29:59,598 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e9390d8c0de9aae718ab828acfcab4f9, NAME => 'testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:29:59,600 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,601 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:29:59,601 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,601 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,603 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,604 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9390d8c0de9aae718ab828acfcab4f9 columnFamilyName a 2024-12-01T14:29:59,604 DEBUG [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,605 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(327): Store=e9390d8c0de9aae718ab828acfcab4f9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,605 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,606 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9390d8c0de9aae718ab828acfcab4f9 columnFamilyName b 2024-12-01T14:29:59,606 DEBUG [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,607 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(327): Store=e9390d8c0de9aae718ab828acfcab4f9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,607 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,607 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9390d8c0de9aae718ab828acfcab4f9 columnFamilyName c 2024-12-01T14:29:59,607 DEBUG [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:29:59,608 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(327): Store=e9390d8c0de9aae718ab828acfcab4f9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:29:59,608 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,609 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,610 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,611 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,611 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,612 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:29:59,614 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,615 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9390d8c0de9aae718ab828acfcab4f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61706966, jitterRate=-0.08049455285072327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:29:59,615 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:29:59,616 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9390d8c0de9aae718ab828acfcab4f9: Running coprocessor pre-open hook at 1733063399601Writing region info on filesystem at 1733063399601Initializing all the Stores at 1733063399602 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399602Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399602Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063399602Cleaning up temporary data from old regions at 1733063399611 (+9 ms)Running coprocessor post-open hooks at 1733063399615 (+4 ms)Region opened successfully at 1733063399616 (+1 ms) 2024-12-01T14:29:59,629 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e9390d8c0de9aae718ab828acfcab4f9 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-01T14:29:59,630 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e9390d8c0de9aae718ab828acfcab4f9/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:00,631 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e9390d8c0de9aae718ab828acfcab4f9/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:00,730 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-01T14:30:00,730 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-01T14:30:00,731 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-01T14:30:00,731 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-01T14:30:00,732 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-01T14:30:00,732 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-01T14:30:00,733 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-01T14:30:00,733 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-01T14:30:00,733 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-01T14:30:00,733 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-01T14:30:01,483 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T14:30:01,632 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e9390d8c0de9aae718ab828acfcab4f9/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:02,633 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e9390d8c0de9aae718ab828acfcab4f9/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:03,634 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e9390d8c0de9aae718ab828acfcab4f9/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:04,634 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e9390d8c0de9aae718ab828acfcab4f9/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:05,635 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e9390d8c0de9aae718ab828acfcab4f9/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:06,635 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e9390d8c0de9aae718ab828acfcab4f9/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:07,637 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e9390d8c0de9aae718ab828acfcab4f9/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:08,638 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e9390d8c0de9aae718ab828acfcab4f9/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:08,641 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e9390d8c0de9aae718ab828acfcab4f9: 2024-12-01T14:30:08,642 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. 2024-12-01T14:30:08,660 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e9390d8c0de9aae718ab828acfcab4f9: 2024-12-01T14:30:08,660 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-01T14:30:08,660 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e9390d8c0de9aae718ab828acfcab4f9, disabling compactions & flushes 2024-12-01T14:30:08,660 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. 2024-12-01T14:30:08,660 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. 2024-12-01T14:30:08,660 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. after waiting 0 ms 2024-12-01T14:30:08,660 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. 2024-12-01T14:30:08,661 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. 2024-12-01T14:30:08,661 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. 2024-12-01T14:30:08,661 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e9390d8c0de9aae718ab828acfcab4f9: Waiting for close lock at 1733063408660Running coprocessor pre-close hooks at 1733063408660Disabling compacts and flushes for region at 1733063408660Disabling writes for close at 1733063408660Writing region close event to WAL at 1733063408661 (+1 ms)Running coprocessor post-close hooks at 1733063408661Closed at 1733063408661 2024-12-01T14:30:08,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741903_1081 (size=2691) 2024-12-01T14:30:08,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741903_1081 (size=2691) 2024-12-01T14:30:08,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741903_1081 (size=2691) 2024-12-01T14:30:08,679 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508, size=2.6 K (2691bytes) 2024-12-01T14:30:08,679 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508 2024-12-01T14:30:08,679 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508 after 0ms 2024-12-01T14:30:08,682 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:08,682 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508 took 3ms 2024-12-01T14:30:08,684 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508 so closing down 2024-12-01T14:30:08,684 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:30:08,685 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733063399508.temp 2024-12-01T14:30:08,686 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000004-wal.1733063399508.temp 2024-12-01T14:30:08,686 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:30:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741904_1082 (size=2094) 2024-12-01T14:30:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741904_1082 (size=2094) 2024-12-01T14:30:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741904_1082 (size=2094) 2024-12-01T14:30:08,698 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000004-wal.1733063399508.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-01T14:30:08,699 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000004-wal.1733063399508.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000026 2024-12-01T14:30:08,699 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-12-01T14:30:08,700 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508, journal: Splitting hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508, size=2.6 K (2691bytes) at 1733063408679Finishing writing output for hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508 so closing down at 1733063408684 (+5 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000004-wal.1733063399508.temp at 1733063408686 (+2 ms)3 split writer threads finished at 1733063408686Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000004-wal.1733063399508.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733063408698 (+12 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000004-wal.1733063399508.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000026 at 1733063408699 (+1 ms)Processed 23 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1733063408699 2024-12-01T14:30:08,701 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063399508 to hdfs://localhost:40889/hbase/oldWALs/wal.1733063399508 2024-12-01T14:30:08,702 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000026 2024-12-01T14:30:08,703 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:30:08,704 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:08,717 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063408705, exclude list is [], retry=0 2024-12-01T14:30:08,719 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:08,720 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:08,720 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:08,722 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063408705 2024-12-01T14:30:08,722 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:30:08,722 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e9390d8c0de9aae718ab828acfcab4f9, NAME => 'testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:30:08,723 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,723 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:08,723 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,723 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,725 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,726 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9390d8c0de9aae718ab828acfcab4f9 columnFamilyName a 2024-12-01T14:30:08,726 DEBUG [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:08,726 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(327): Store=e9390d8c0de9aae718ab828acfcab4f9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:08,726 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,727 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9390d8c0de9aae718ab828acfcab4f9 columnFamilyName b 2024-12-01T14:30:08,727 DEBUG [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:08,727 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(327): Store=e9390d8c0de9aae718ab828acfcab4f9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:08,728 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,728 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9390d8c0de9aae718ab828acfcab4f9 columnFamilyName c 2024-12-01T14:30:08,728 DEBUG [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:08,729 INFO [StoreOpener-e9390d8c0de9aae718ab828acfcab4f9-1 {}] regionserver.HStore(327): Store=e9390d8c0de9aae718ab828acfcab4f9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:08,729 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,729 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,731 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,731 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000026 2024-12-01T14:30:08,733 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:08,734 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000026 2024-12-01T14:30:08,734 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e9390d8c0de9aae718ab828acfcab4f9 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-01T14:30:08,750 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/.tmp/a/ecbfee56313f4a34af9c46d022b99452 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733063408652/Put/seqid=0 2024-12-01T14:30:08,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741906_1084 (size=5523) 2024-12-01T14:30:08,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741906_1084 (size=5523) 2024-12-01T14:30:08,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741906_1084 (size=5523) 2024-12-01T14:30:08,756 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/.tmp/a/ecbfee56313f4a34af9c46d022b99452 2024-12-01T14:30:08,776 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/.tmp/b/1a9081f2a2db4521a16cb633a32e47cb is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733063408643/Put/seqid=0 2024-12-01T14:30:08,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741907_1085 (size=5524) 2024-12-01T14:30:08,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741907_1085 (size=5524) 2024-12-01T14:30:08,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741907_1085 (size=5524) 2024-12-01T14:30:08,782 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/.tmp/b/1a9081f2a2db4521a16cb633a32e47cb 2024-12-01T14:30:08,802 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/.tmp/c/2f10ee2dfdfc4ecca5e36a9a180ca9a1 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733063408650/Put/seqid=0 2024-12-01T14:30:08,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741908_1086 (size=5457) 2024-12-01T14:30:08,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741908_1086 (size=5457) 2024-12-01T14:30:08,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741908_1086 (size=5457) 2024-12-01T14:30:08,870 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/.tmp/c/2f10ee2dfdfc4ecca5e36a9a180ca9a1 2024-12-01T14:30:08,878 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/.tmp/a/ecbfee56313f4a34af9c46d022b99452 as hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/a/ecbfee56313f4a34af9c46d022b99452 2024-12-01T14:30:08,886 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/a/ecbfee56313f4a34af9c46d022b99452, entries=7, sequenceid=26, filesize=5.4 K 2024-12-01T14:30:08,887 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/.tmp/b/1a9081f2a2db4521a16cb633a32e47cb as hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/b/1a9081f2a2db4521a16cb633a32e47cb 2024-12-01T14:30:08,893 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/b/1a9081f2a2db4521a16cb633a32e47cb, entries=7, sequenceid=26, filesize=5.4 K 2024-12-01T14:30:08,894 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/.tmp/c/2f10ee2dfdfc4ecca5e36a9a180ca9a1 as hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/c/2f10ee2dfdfc4ecca5e36a9a180ca9a1 2024-12-01T14:30:08,900 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/c/2f10ee2dfdfc4ecca5e36a9a180ca9a1, entries=6, sequenceid=26, filesize=5.3 K 2024-12-01T14:30:08,900 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for e9390d8c0de9aae718ab828acfcab4f9 in 166ms, sequenceid=26, compaction requested=false; wal=null 2024-12-01T14:30:08,901 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/0000000000000000026 2024-12-01T14:30:08,902 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,903 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,903 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:30:08,905 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,907 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsAfterAbortingFlush/e9390d8c0de9aae718ab828acfcab4f9/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-01T14:30:08,908 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9390d8c0de9aae718ab828acfcab4f9; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67496552, jitterRate=0.00577700138092041}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:30:08,908 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e9390d8c0de9aae718ab828acfcab4f9 2024-12-01T14:30:08,909 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9390d8c0de9aae718ab828acfcab4f9: Running coprocessor pre-open hook at 1733063408723Writing region info on filesystem at 1733063408723Initializing all the Stores at 1733063408724 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063408724Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063408724Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063408724Obtaining lock to block concurrent updates at 1733063408734 (+10 ms)Preparing flush snapshotting stores in e9390d8c0de9aae718ab828acfcab4f9 at 1733063408734Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733063408735 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1733063399452.e9390d8c0de9aae718ab828acfcab4f9. at 1733063408735Flushing e9390d8c0de9aae718ab828acfcab4f9/a: creating writer at 1733063408735Flushing e9390d8c0de9aae718ab828acfcab4f9/a: appending metadata at 1733063408749 (+14 ms)Flushing e9390d8c0de9aae718ab828acfcab4f9/a: closing flushed file at 1733063408749Flushing e9390d8c0de9aae718ab828acfcab4f9/b: creating writer at 1733063408761 (+12 ms)Flushing e9390d8c0de9aae718ab828acfcab4f9/b: appending metadata at 1733063408775 (+14 ms)Flushing e9390d8c0de9aae718ab828acfcab4f9/b: closing flushed file at 1733063408775Flushing e9390d8c0de9aae718ab828acfcab4f9/c: creating writer at 1733063408787 (+12 ms)Flushing e9390d8c0de9aae718ab828acfcab4f9/c: appending metadata at 1733063408801 (+14 ms)Flushing e9390d8c0de9aae718ab828acfcab4f9/c: closing flushed file at 1733063408801Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3631110a: reopening flushed file at 1733063408877 (+76 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25eaa3e2: reopening flushed file at 1733063408886 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a49ccba: reopening flushed file at 1733063408894 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for e9390d8c0de9aae718ab828acfcab4f9 in 166ms, sequenceid=26, compaction requested=false; wal=null at 1733063408900 (+6 ms)Cleaning up temporary data from old regions at 1733063408903 (+3 ms)Running coprocessor post-open hooks at 1733063408908 (+5 ms)Region opened successfully at 1733063408909 (+1 ms) 2024-12-01T14:30:08,927 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=422 (was 424), OpenFileDescriptor=1183 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=177 (was 200), ProcessCount=11 (was 11), AvailableMemoryMB=4904 (was 4920) 2024-12-01T14:30:08,927 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-01T14:30:08,938 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=422, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=177, ProcessCount=11, AvailableMemoryMB=4903 2024-12-01T14:30:08,938 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-01T14:30:08,954 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:30:08,955 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:30:08,956 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:30:08,958 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-48236744, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/hregion-48236744, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:08,969 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-48236744/hregion-48236744.1733063408958, exclude list is [], retry=0 2024-12-01T14:30:08,972 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:08,972 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:08,972 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:08,974 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-48236744/hregion-48236744.1733063408958 2024-12-01T14:30:08,974 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:30:08,975 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a297a9a1e1021f5db017b307b296f101, NAME => 'testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/hbase 2024-12-01T14:30:08,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741910_1088 (size=61) 2024-12-01T14:30:08,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741910_1088 (size=61) 2024-12-01T14:30:08,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741910_1088 (size=61) 2024-12-01T14:30:08,984 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:08,985 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:08,986 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a297a9a1e1021f5db017b307b296f101 columnFamilyName a 2024-12-01T14:30:08,986 DEBUG [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:08,986 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] regionserver.HStore(327): Store=a297a9a1e1021f5db017b307b296f101/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:08,986 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:08,987 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:08,987 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:08,988 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:08,988 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:08,989 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:08,991 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:30:08,991 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a297a9a1e1021f5db017b307b296f101; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69989960, jitterRate=0.04293167591094971}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:30:08,992 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a297a9a1e1021f5db017b307b296f101: Writing region info on filesystem at 1733063408984Initializing all the Stores at 1733063408984Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063408984Cleaning up temporary data from old regions at 1733063408988 (+4 ms)Region opened successfully at 1733063408992 (+4 ms) 2024-12-01T14:30:08,992 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a297a9a1e1021f5db017b307b296f101, disabling compactions & flushes 2024-12-01T14:30:08,992 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. 2024-12-01T14:30:08,992 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. 2024-12-01T14:30:08,992 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. after waiting 0 ms 2024-12-01T14:30:08,992 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. 2024-12-01T14:30:08,993 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. 2024-12-01T14:30:08,993 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a297a9a1e1021f5db017b307b296f101: Waiting for close lock at 1733063408992Disabling compacts and flushes for region at 1733063408992Disabling writes for close at 1733063408992Writing region close event to WAL at 1733063408993 (+1 ms)Closed at 1733063408993 2024-12-01T14:30:08,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741909_1087 (size=95) 2024-12-01T14:30:08,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741909_1087 (size=95) 2024-12-01T14:30:08,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741909_1087 (size=95) 2024-12-01T14:30:08,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:30:08,997 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-48236744:(num 1733063408958) 2024-12-01T14:30:08,997 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:30:08,999 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:09,011 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999, exclude list is [], retry=0 2024-12-01T14:30:09,013 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:09,014 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:09,014 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:09,016 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999 2024-12-01T14:30:09,016 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:30:09,016 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a297a9a1e1021f5db017b307b296f101, NAME => 'testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:30:09,017 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:09,017 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,017 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,018 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,019 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a297a9a1e1021f5db017b307b296f101 columnFamilyName a 2024-12-01T14:30:09,019 DEBUG [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:09,019 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] regionserver.HStore(327): Store=a297a9a1e1021f5db017b307b296f101/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:09,019 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,020 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,022 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,022 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,023 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,024 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a297a9a1e1021f5db017b307b296f101; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58848184, jitterRate=-0.12309372425079346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:30:09,025 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a297a9a1e1021f5db017b307b296f101: Writing region info on filesystem at 1733063409017Initializing all the Stores at 1733063409018 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063409018Cleaning up temporary data from old regions at 1733063409022 (+4 ms)Region opened successfully at 1733063409024 (+2 ms) 2024-12-01T14:30:09,032 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a297a9a1e1021f5db017b307b296f101, disabling compactions & flushes 2024-12-01T14:30:09,032 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. 2024-12-01T14:30:09,032 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. 2024-12-01T14:30:09,032 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. after waiting 0 ms 2024-12-01T14:30:09,032 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. 2024-12-01T14:30:09,032 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. 2024-12-01T14:30:09,032 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. 2024-12-01T14:30:09,032 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a297a9a1e1021f5db017b307b296f101: Waiting for close lock at 1733063409032Disabling compacts and flushes for region at 1733063409032Disabling writes for close at 1733063409032Writing region close event to WAL at 1733063409032Closed at 1733063409032 2024-12-01T14:30:09,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741911_1089 (size=1050) 2024-12-01T14:30:09,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741911_1089 (size=1050) 2024-12-01T14:30:09,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741911_1089 (size=1050) 2024-12-01T14:30:09,049 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999, size=1.0 K (1050bytes) 2024-12-01T14:30:09,049 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999 2024-12-01T14:30:09,050 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999 after 1ms 2024-12-01T14:30:09,051 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:09,052 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999 took 3ms 2024-12-01T14:30:09,054 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999 so closing down 2024-12-01T14:30:09,054 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:30:09,055 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733063408999.temp 2024-12-01T14:30:09,056 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000003-wal.1733063408999.temp 2024-12-01T14:30:09,056 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:30:09,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741912_1090 (size=1050) 2024-12-01T14:30:09,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741912_1090 (size=1050) 2024-12-01T14:30:09,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741912_1090 (size=1050) 2024-12-01T14:30:09,063 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000003-wal.1733063408999.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-01T14:30:09,064 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000003-wal.1733063408999.temp to hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000012 2024-12-01T14:30:09,064 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-12-01T14:30:09,064 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999, journal: Splitting hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999, size=1.0 K (1050bytes) at 1733063409049Finishing writing output for hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999 so closing down at 1733063409054 (+5 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000003-wal.1733063408999.temp at 1733063409056 (+2 ms)3 split writer threads finished at 1733063409056Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000003-wal.1733063408999.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733063409063 (+7 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000003-wal.1733063408999.temp to hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000012 at 1733063409064 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1733063409064 2024-12-01T14:30:09,066 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063408999 to hdfs://localhost:40889/hbase/oldWALs/wal.1733063408999 2024-12-01T14:30:09,067 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000012 2024-12-01T14:30:09,071 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:09,381 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:30:09,383 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:09,394 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063409384, exclude list is [], retry=0 2024-12-01T14:30:09,396 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:09,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:09,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:09,398 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063409384 2024-12-01T14:30:09,398 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:30:09,399 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a297a9a1e1021f5db017b307b296f101, NAME => 'testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:30:09,399 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:09,399 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,399 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,401 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,403 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a297a9a1e1021f5db017b307b296f101 columnFamilyName a 2024-12-01T14:30:09,403 DEBUG [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:09,404 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] regionserver.HStore(327): Store=a297a9a1e1021f5db017b307b296f101/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:09,404 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,405 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,406 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,407 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000012 2024-12-01T14:30:09,409 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:09,410 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000012 2024-12-01T14:30:09,411 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a297a9a1e1021f5db017b307b296f101 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-01T14:30:09,429 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/.tmp/a/aacc71cf2e5645bb9a5cb24f86d87413 is 79, key is testDatalossWhenInputError/a:x0/1733063409025/Put/seqid=0 2024-12-01T14:30:09,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741914_1092 (size=5808) 2024-12-01T14:30:09,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741914_1092 (size=5808) 2024-12-01T14:30:09,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741914_1092 (size=5808) 2024-12-01T14:30:09,436 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/.tmp/a/aacc71cf2e5645bb9a5cb24f86d87413 2024-12-01T14:30:09,445 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/.tmp/a/aacc71cf2e5645bb9a5cb24f86d87413 as hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/a/aacc71cf2e5645bb9a5cb24f86d87413 2024-12-01T14:30:09,453 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/a/aacc71cf2e5645bb9a5cb24f86d87413, entries=10, sequenceid=12, filesize=5.7 K 2024-12-01T14:30:09,454 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for a297a9a1e1021f5db017b307b296f101 in 43ms, sequenceid=12, compaction requested=false; wal=null 2024-12-01T14:30:09,455 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/0000000000000000012 2024-12-01T14:30:09,456 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,456 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,458 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,461 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-01T14:30:09,462 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a297a9a1e1021f5db017b307b296f101; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71798190, jitterRate=0.06987640261650085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:30:09,463 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a297a9a1e1021f5db017b307b296f101: Writing region info on filesystem at 1733063409399Initializing all the Stores at 1733063409401 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063409401Obtaining lock to block concurrent updates at 1733063409411 (+10 ms)Preparing flush snapshotting stores in a297a9a1e1021f5db017b307b296f101 at 1733063409411Finished memstore snapshotting testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733063409411Flushing stores of testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101. at 1733063409411Flushing a297a9a1e1021f5db017b307b296f101/a: creating writer at 1733063409411Flushing a297a9a1e1021f5db017b307b296f101/a: appending metadata at 1733063409429 (+18 ms)Flushing a297a9a1e1021f5db017b307b296f101/a: closing flushed file at 1733063409429Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bc80f48: reopening flushed file at 1733063409443 (+14 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for a297a9a1e1021f5db017b307b296f101 in 43ms, sequenceid=12, compaction requested=false; wal=null at 1733063409454 (+11 ms)Cleaning up temporary data from old regions at 1733063409456 (+2 ms)Region opened successfully at 1733063409463 (+7 ms) 2024-12-01T14:30:09,466 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a297a9a1e1021f5db017b307b296f101, NAME => 'testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:30:09,466 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733063408954.a297a9a1e1021f5db017b307b296f101.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:09,466 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,466 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,468 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,468 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a297a9a1e1021f5db017b307b296f101 columnFamilyName a 2024-12-01T14:30:09,468 DEBUG [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:09,474 DEBUG [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/a/aacc71cf2e5645bb9a5cb24f86d87413 2024-12-01T14:30:09,474 INFO [StoreOpener-a297a9a1e1021f5db017b307b296f101-1 {}] regionserver.HStore(327): Store=a297a9a1e1021f5db017b307b296f101/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:09,474 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,475 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,476 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,477 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,477 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,479 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a297a9a1e1021f5db017b307b296f101 2024-12-01T14:30:09,482 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testDatalossWhenInputError/a297a9a1e1021f5db017b307b296f101/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-01T14:30:09,483 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a297a9a1e1021f5db017b307b296f101; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72991138, jitterRate=0.08765271306037903}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T14:30:09,483 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a297a9a1e1021f5db017b307b296f101: Writing region info on filesystem at 1733063409466Initializing all the Stores at 1733063409467 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063409467Cleaning up temporary data from old regions at 1733063409477 (+10 ms)Region opened successfully at 1733063409483 (+6 ms) 2024-12-01T14:30:09,501 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=432 (was 422) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:40222 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:47782 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:58672 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:58600 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1542046065-172.17.0.3-1733063375577:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:40152 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1265 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=177 (was 177), ProcessCount=11 (was 11), AvailableMemoryMB=4890 (was 4903) 2024-12-01T14:30:09,501 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1265 is superior to 1024 2024-12-01T14:30:09,513 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=432, OpenFileDescriptor=1265, MaxFileDescriptor=1048576, SystemLoadAverage=177, ProcessCount=11, AvailableMemoryMB=4889 2024-12-01T14:30:09,513 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1265 is superior to 1024 2024-12-01T14:30:09,527 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:30:09,529 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:30:09,530 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:30:09,533 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-07006614, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/hregion-07006614, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:09,545 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-07006614/hregion-07006614.1733063409533, exclude list is [], retry=0 2024-12-01T14:30:09,548 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:09,548 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:09,548 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:09,551 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-07006614/hregion-07006614.1733063409533 2024-12-01T14:30:09,551 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:30:09,551 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ea9d6f17a69c02b8582083110ec2c6be, NAME => 'testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/hbase 2024-12-01T14:30:09,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741916_1094 (size=63) 2024-12-01T14:30:09,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741916_1094 (size=63) 2024-12-01T14:30:09,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741916_1094 (size=63) 2024-12-01T14:30:09,559 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:09,560 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,562 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9d6f17a69c02b8582083110ec2c6be columnFamilyName a 2024-12-01T14:30:09,562 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:09,562 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(327): Store=ea9d6f17a69c02b8582083110ec2c6be/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:09,563 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,564 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9d6f17a69c02b8582083110ec2c6be columnFamilyName b 2024-12-01T14:30:09,564 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:09,564 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(327): Store=ea9d6f17a69c02b8582083110ec2c6be/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:09,565 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,566 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9d6f17a69c02b8582083110ec2c6be columnFamilyName c 2024-12-01T14:30:09,566 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:09,567 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(327): Store=ea9d6f17a69c02b8582083110ec2c6be/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:09,567 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,569 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,569 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,570 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:30:09,571 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,575 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:30:09,575 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea9d6f17a69c02b8582083110ec2c6be; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66964521, jitterRate=-0.0021508783102035522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:30:09,576 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea9d6f17a69c02b8582083110ec2c6be: Writing region info on filesystem at 1733063409559Initializing all the Stores at 1733063409560 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063409560Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063409560Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063409560Cleaning up temporary data from old regions at 1733063409569 (+9 ms)Region opened successfully at 1733063409576 (+7 ms) 2024-12-01T14:30:09,576 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea9d6f17a69c02b8582083110ec2c6be, disabling compactions & flushes 2024-12-01T14:30:09,576 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. 2024-12-01T14:30:09,576 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. 2024-12-01T14:30:09,576 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. after waiting 0 ms 2024-12-01T14:30:09,576 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. 2024-12-01T14:30:09,576 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. 2024-12-01T14:30:09,576 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea9d6f17a69c02b8582083110ec2c6be: Waiting for close lock at 1733063409576Disabling compacts and flushes for region at 1733063409576Disabling writes for close at 1733063409576Writing region close event to WAL at 1733063409576Closed at 1733063409576 2024-12-01T14:30:09,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741915_1093 (size=95) 2024-12-01T14:30:09,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741915_1093 (size=95) 2024-12-01T14:30:09,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741915_1093 (size=95) 2024-12-01T14:30:09,581 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:30:09,581 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-07006614:(num 1733063409533) 2024-12-01T14:30:09,581 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:30:09,583 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:09,583 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T14:30:09,595 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583, exclude list is [], retry=0 2024-12-01T14:30:09,598 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:09,598 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:09,598 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:09,600 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 2024-12-01T14:30:09,600 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:40289:40289)] 2024-12-01T14:30:09,601 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea9d6f17a69c02b8582083110ec2c6be, NAME => 'testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:30:09,601 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:09,601 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,601 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,603 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,603 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9d6f17a69c02b8582083110ec2c6be columnFamilyName a 2024-12-01T14:30:09,604 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:09,604 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(327): Store=ea9d6f17a69c02b8582083110ec2c6be/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:09,604 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,605 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9d6f17a69c02b8582083110ec2c6be columnFamilyName b 2024-12-01T14:30:09,605 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:09,605 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(327): Store=ea9d6f17a69c02b8582083110ec2c6be/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:09,606 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,606 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9d6f17a69c02b8582083110ec2c6be columnFamilyName c 2024-12-01T14:30:09,606 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:09,607 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(327): Store=ea9d6f17a69c02b8582083110ec2c6be/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:09,607 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,607 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,609 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,610 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,610 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,610 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:30:09,612 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:09,612 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea9d6f17a69c02b8582083110ec2c6be; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71441411, jitterRate=0.06455998122692108}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:30:09,613 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea9d6f17a69c02b8582083110ec2c6be: Writing region info on filesystem at 1733063409601Initializing all the Stores at 1733063409602 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063409602Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063409602Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063409602Cleaning up temporary data from old regions at 1733063409610 (+8 ms)Region opened successfully at 1733063409613 (+3 ms) 2024-12-01T14:30:09,617 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733063409616/Put/seqid=0 2024-12-01T14:30:09,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741918_1096 (size=4875) 2024-12-01T14:30:09,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741918_1096 (size=4875) 2024-12-01T14:30:09,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741918_1096 (size=4875) 2024-12-01T14:30:09,627 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733063409627/Put/seqid=0 2024-12-01T14:30:09,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741919_1097 (size=4875) 2024-12-01T14:30:09,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741919_1097 (size=4875) 2024-12-01T14:30:09,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741919_1097 (size=4875) 2024-12-01T14:30:09,637 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733063409637/Put/seqid=0 2024-12-01T14:30:09,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741920_1098 (size=4875) 2024-12-01T14:30:09,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741920_1098 (size=4875) 2024-12-01T14:30:09,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741920_1098 (size=4875) 2024-12-01T14:30:09,644 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in ea9d6f17a69c02b8582083110ec2c6be/a 2024-12-01T14:30:09,647 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-01T14:30:09,647 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T14:30:09,648 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in ea9d6f17a69c02b8582083110ec2c6be/a 2024-12-01T14:30:09,651 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-01T14:30:09,651 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T14:30:09,651 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in ea9d6f17a69c02b8582083110ec2c6be/a 2024-12-01T14:30:09,654 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-01T14:30:09,654 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-01T14:30:09,654 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea9d6f17a69c02b8582083110ec2c6be 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-01T14:30:09,669 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/.tmp/a/cae151ed8b0a4148aad954df28504627 is 55, key is testCompactedBulkLoadedFiles/a:a/1733063409613/Put/seqid=0 2024-12-01T14:30:09,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741921_1099 (size=5107) 2024-12-01T14:30:09,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741921_1099 (size=5107) 2024-12-01T14:30:09,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741921_1099 (size=5107) 2024-12-01T14:30:09,676 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/.tmp/a/cae151ed8b0a4148aad954df28504627 2024-12-01T14:30:09,681 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/.tmp/a/cae151ed8b0a4148aad954df28504627 as hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/cae151ed8b0a4148aad954df28504627 2024-12-01T14:30:09,686 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/cae151ed8b0a4148aad954df28504627, entries=1, sequenceid=4, filesize=5.0 K 2024-12-01T14:30:09,687 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for ea9d6f17a69c02b8582083110ec2c6be in 33ms, sequenceid=4, compaction requested=false 2024-12-01T14:30:09,687 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ea9d6f17a69c02b8582083110ec2c6be: 2024-12-01T14:30:09,689 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/0466788aba964830a88f0390e1f36d05_SeqId_4_ 2024-12-01T14:30:09,690 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ac9f9c959d17473e948f93783696a82c_SeqId_4_ 2024-12-01T14:30:09,691 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/f220672f7cd84db58f6133efd0adbcc4_SeqId_4_ 2024-12-01T14:30:09,691 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile0 into ea9d6f17a69c02b8582083110ec2c6be/a as hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/0466788aba964830a88f0390e1f36d05_SeqId_4_ - updating store file list. 2024-12-01T14:30:09,695 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0466788aba964830a88f0390e1f36d05_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T14:30:09,696 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/0466788aba964830a88f0390e1f36d05_SeqId_4_ into ea9d6f17a69c02b8582083110ec2c6be/a 2024-12-01T14:30:09,696 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile0 into ea9d6f17a69c02b8582083110ec2c6be/a (new location: hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/0466788aba964830a88f0390e1f36d05_SeqId_4_) 2024-12-01T14:30:09,697 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile1 into ea9d6f17a69c02b8582083110ec2c6be/a as hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ac9f9c959d17473e948f93783696a82c_SeqId_4_ - updating store file list. 2024-12-01T14:30:09,701 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ac9f9c959d17473e948f93783696a82c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T14:30:09,701 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ac9f9c959d17473e948f93783696a82c_SeqId_4_ into ea9d6f17a69c02b8582083110ec2c6be/a 2024-12-01T14:30:09,701 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile1 into ea9d6f17a69c02b8582083110ec2c6be/a (new location: hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ac9f9c959d17473e948f93783696a82c_SeqId_4_) 2024-12-01T14:30:09,702 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile2 into ea9d6f17a69c02b8582083110ec2c6be/a as hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/f220672f7cd84db58f6133efd0adbcc4_SeqId_4_ - updating store file list. 2024-12-01T14:30:09,706 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for f220672f7cd84db58f6133efd0adbcc4_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T14:30:09,706 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/f220672f7cd84db58f6133efd0adbcc4_SeqId_4_ into ea9d6f17a69c02b8582083110ec2c6be/a 2024-12-01T14:30:09,706 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40889/hbase/testCompactedBulkLoadedFiles/hfile2 into ea9d6f17a69c02b8582083110ec2c6be/a (new location: hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/f220672f7cd84db58f6133efd0adbcc4_SeqId_4_) 2024-12-01T14:30:09,716 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-01T14:30:09,716 DEBUG [Time-limited test {}] regionserver.HStore(1541): ea9d6f17a69c02b8582083110ec2c6be/a is initiating major compaction (all files) 2024-12-01T14:30:09,716 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ea9d6f17a69c02b8582083110ec2c6be/a in testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. 2024-12-01T14:30:09,716 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/cae151ed8b0a4148aad954df28504627, hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/0466788aba964830a88f0390e1f36d05_SeqId_4_, hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ac9f9c959d17473e948f93783696a82c_SeqId_4_, hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/f220672f7cd84db58f6133efd0adbcc4_SeqId_4_] into tmpdir=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/.tmp, totalSize=19.3 K 2024-12-01T14:30:09,717 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cae151ed8b0a4148aad954df28504627, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733063409613 2024-12-01T14:30:09,717 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0466788aba964830a88f0390e1f36d05_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-01T14:30:09,718 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ac9f9c959d17473e948f93783696a82c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-01T14:30:09,718 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f220672f7cd84db58f6133efd0adbcc4_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-01T14:30:09,730 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/.tmp/a/ce44c242d0da4ee589eb514840d8d4e8 is 55, key is testCompactedBulkLoadedFiles/a:a/1733063409613/Put/seqid=0 2024-12-01T14:30:09,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741922_1100 (size=6154) 2024-12-01T14:30:09,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741922_1100 (size=6154) 2024-12-01T14:30:09,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741922_1100 (size=6154) 2024-12-01T14:30:09,745 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/.tmp/a/ce44c242d0da4ee589eb514840d8d4e8 as hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ce44c242d0da4ee589eb514840d8d4e8 2024-12-01T14:30:09,753 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in ea9d6f17a69c02b8582083110ec2c6be/a of ea9d6f17a69c02b8582083110ec2c6be into ce44c242d0da4ee589eb514840d8d4e8(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-01T14:30:09,753 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ea9d6f17a69c02b8582083110ec2c6be: 2024-12-01T14:30:09,753 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-01T14:30:09,753 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-01T14:30:09,788 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583, size=0 (0bytes) 2024-12-01T14:30:09,788 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 might be still open, length is 0 2024-12-01T14:30:09,788 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 2024-12-01T14:30:09,789 WARN [IPC Server handler 4 on default port 40889 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-01T14:30:09,789 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 after 1ms 2024-12-01T14:30:10,730 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-01T14:30:10,730 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-01T14:30:10,733 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-01T14:30:10,733 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-01T14:30:12,490 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:47828 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33423:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47828 dst: /127.0.0.1:33423 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33423 remote=/127.0.0.1:47828]. Total timeout mills is 60000, 57263 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:12,491 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:40264 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40063:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40264 dst: /127.0.0.1:40063 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:12,491 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:58696 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33303:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58696 dst: /127.0.0.1:33303 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:12,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741917_1101 (size=1171) 2024-12-01T14:30:12,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741917_1101 (size=1171) 2024-12-01T14:30:12,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741917_1101 (size=1171) 2024-12-01T14:30:13,790 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 after 4002ms 2024-12-01T14:30:13,792 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:13,792 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 took 4004ms 2024-12-01T14:30:13,794 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583; continuing. 2024-12-01T14:30:13,794 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 so closing down 2024-12-01T14:30:13,794 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:30:13,795 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733063409583.temp 2024-12-01T14:30:13,796 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000003-wal.1733063409583.temp 2024-12-01T14:30:13,797 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:30:13,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741923_1102 (size=548) 2024-12-01T14:30:13,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741923_1102 (size=548) 2024-12-01T14:30:13,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741923_1102 (size=548) 2024-12-01T14:30:13,878 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000003-wal.1733063409583.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-01T14:30:13,879 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000003-wal.1733063409583.temp to hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000008 2024-12-01T14:30:13,879 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 86 ms; skipped=3; WAL=hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T14:30:13,879 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583, journal: Splitting hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583, size=0 (0bytes) at 1733063409788Finishing writing output for hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 so closing down at 1733063413794 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000003-wal.1733063409583.temp at 1733063413796 (+2 ms)3 split writer threads finished at 1733063413797 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000003-wal.1733063409583.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733063413878 (+81 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000003-wal.1733063409583.temp to hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000008 at 1733063413879 (+1 ms)Processed 5 edits across 1 Regions in 86 ms; skipped=3; WAL=hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583, size=0, length=0, corrupted=false, cancelled=false at 1733063413879 2024-12-01T14:30:13,880 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 to hdfs://localhost:40889/hbase/oldWALs/wal.1733063409583 2024-12-01T14:30:13,881 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000008 2024-12-01T14:30:13,881 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:30:13,883 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:13,894 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063413883, exclude list is [], retry=0 2024-12-01T14:30:13,896 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:13,897 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:13,897 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:13,898 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063413883 2024-12-01T14:30:13,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:40289:40289)] 2024-12-01T14:30:13,899 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea9d6f17a69c02b8582083110ec2c6be, NAME => 'testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:30:13,899 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:13,899 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,899 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,900 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,901 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9d6f17a69c02b8582083110ec2c6be columnFamilyName a 2024-12-01T14:30:13,901 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:13,907 DEBUG [StoreFileOpener-ea9d6f17a69c02b8582083110ec2c6be-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0466788aba964830a88f0390e1f36d05_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T14:30:13,907 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/0466788aba964830a88f0390e1f36d05_SeqId_4_ 2024-12-01T14:30:13,910 DEBUG [StoreFileOpener-ea9d6f17a69c02b8582083110ec2c6be-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ac9f9c959d17473e948f93783696a82c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T14:30:13,911 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ac9f9c959d17473e948f93783696a82c_SeqId_4_ 2024-12-01T14:30:13,915 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/cae151ed8b0a4148aad954df28504627 2024-12-01T14:30:13,920 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ce44c242d0da4ee589eb514840d8d4e8 2024-12-01T14:30:13,924 DEBUG [StoreFileOpener-ea9d6f17a69c02b8582083110ec2c6be-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for f220672f7cd84db58f6133efd0adbcc4_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-01T14:30:13,924 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/f220672f7cd84db58f6133efd0adbcc4_SeqId_4_ 2024-12-01T14:30:13,924 WARN [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/0466788aba964830a88f0390e1f36d05_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@79638bd7 2024-12-01T14:30:13,924 WARN [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ac9f9c959d17473e948f93783696a82c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@79638bd7 2024-12-01T14:30:13,924 WARN [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/cae151ed8b0a4148aad954df28504627 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@79638bd7 2024-12-01T14:30:13,924 WARN [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/f220672f7cd84db58f6133efd0adbcc4_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@79638bd7 2024-12-01T14:30:13,924 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/0466788aba964830a88f0390e1f36d05_SeqId_4_, hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ac9f9c959d17473e948f93783696a82c_SeqId_4_, hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/cae151ed8b0a4148aad954df28504627, hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/f220672f7cd84db58f6133efd0adbcc4_SeqId_4_] to archive 2024-12-01T14:30:13,925 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-01T14:30:13,927 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/0466788aba964830a88f0390e1f36d05_SeqId_4_ to hdfs://localhost:40889/hbase/archive/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/0466788aba964830a88f0390e1f36d05_SeqId_4_ 2024-12-01T14:30:13,928 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ac9f9c959d17473e948f93783696a82c_SeqId_4_ to hdfs://localhost:40889/hbase/archive/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/ac9f9c959d17473e948f93783696a82c_SeqId_4_ 2024-12-01T14:30:13,929 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/cae151ed8b0a4148aad954df28504627 to hdfs://localhost:40889/hbase/archive/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/cae151ed8b0a4148aad954df28504627 2024-12-01T14:30:13,930 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/f220672f7cd84db58f6133efd0adbcc4_SeqId_4_ to hdfs://localhost:40889/hbase/archive/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/a/f220672f7cd84db58f6133efd0adbcc4_SeqId_4_ 2024-12-01T14:30:13,930 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(327): Store=ea9d6f17a69c02b8582083110ec2c6be/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:13,931 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,931 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9d6f17a69c02b8582083110ec2c6be columnFamilyName b 2024-12-01T14:30:13,931 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:13,932 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(327): Store=ea9d6f17a69c02b8582083110ec2c6be/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:13,932 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,932 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9d6f17a69c02b8582083110ec2c6be columnFamilyName c 2024-12-01T14:30:13,932 DEBUG [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:13,933 INFO [StoreOpener-ea9d6f17a69c02b8582083110ec2c6be-1 {}] regionserver.HStore(327): Store=ea9d6f17a69c02b8582083110ec2c6be/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:13,933 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,934 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,935 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,935 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000008 2024-12-01T14:30:13,937 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:13,938 DEBUG [Time-limited test {}] regionserver.HRegion(5836): ea9d6f17a69c02b8582083110ec2c6be : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "ea9d6f17a69c02b8582083110ec2c6be" family_name: "a" compaction_input: "cae151ed8b0a4148aad954df28504627" compaction_input: "0466788aba964830a88f0390e1f36d05_SeqId_4_" compaction_input: "ac9f9c959d17473e948f93783696a82c_SeqId_4_" compaction_input: "f220672f7cd84db58f6133efd0adbcc4_SeqId_4_" compaction_output: "ce44c242d0da4ee589eb514840d8d4e8" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-01T14:30:13,938 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-01T14:30:13,938 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000008 2024-12-01T14:30:13,939 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/0000000000000000008 2024-12-01T14:30:13,940 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,940 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,941 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:30:13,942 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea9d6f17a69c02b8582083110ec2c6be 2024-12-01T14:30:13,944 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testCompactedBulkLoadedFiles/ea9d6f17a69c02b8582083110ec2c6be/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-01T14:30:13,944 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea9d6f17a69c02b8582083110ec2c6be; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73722772, jitterRate=0.09855490922927856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:30:13,944 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea9d6f17a69c02b8582083110ec2c6be: Writing region info on filesystem at 1733063413899Initializing all the Stores at 1733063413900 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063413900Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063413900Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063413900Cleaning up temporary data from old regions at 1733063413940 (+40 ms)Region opened successfully at 1733063413944 (+4 ms) 2024-12-01T14:30:13,946 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea9d6f17a69c02b8582083110ec2c6be, disabling compactions & flushes 2024-12-01T14:30:13,946 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. 2024-12-01T14:30:13,947 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. 2024-12-01T14:30:13,947 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. after waiting 0 ms 2024-12-01T14:30:13,947 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. 2024-12-01T14:30:13,947 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733063409528.ea9d6f17a69c02b8582083110ec2c6be. 2024-12-01T14:30:13,947 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea9d6f17a69c02b8582083110ec2c6be: Waiting for close lock at 1733063413946Disabling compacts and flushes for region at 1733063413946Disabling writes for close at 1733063413947 (+1 ms)Writing region close event to WAL at 1733063413947Closed at 1733063413947 2024-12-01T14:30:13,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741924_1103 (size=95) 2024-12-01T14:30:13,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741924_1103 (size=95) 2024-12-01T14:30:13,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741924_1103 (size=95) 2024-12-01T14:30:13,952 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:30:13,952 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733063413883) 2024-12-01T14:30:13,964 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=445 (was 432) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1567494068_22 at /127.0.0.1:57034 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:40889 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1567494068_22 at /127.0.0.1:43338 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:40889 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1567494068_22 at /127.0.0.1:52810 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1265) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=162 (was 177), ProcessCount=11 (was 11), AvailableMemoryMB=4866 (was 4889) 2024-12-01T14:30:13,964 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-01T14:30:13,976 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=445, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=162, ProcessCount=11, AvailableMemoryMB=4865 2024-12-01T14:30:13,976 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-01T14:30:13,988 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:30:13,990 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T14:30:13,991 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T14:30:13,994 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-35393853, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/hregion-35393853, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:14,006 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-35393853/hregion-35393853.1733063413994, exclude list is [], retry=0 2024-12-01T14:30:14,009 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:14,009 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:14,009 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:14,011 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-35393853/hregion-35393853.1733063413994 2024-12-01T14:30:14,011 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:30:14,011 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9bdc134bd4398a426a60f85f3d895c84, NAME => 'testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40889/hbase 2024-12-01T14:30:14,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741926_1105 (size=67) 2024-12-01T14:30:14,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741926_1105 (size=67) 2024-12-01T14:30:14,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741926_1105 (size=67) 2024-12-01T14:30:14,019 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:14,020 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,021 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName a 2024-12-01T14:30:14,021 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:14,022 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:14,022 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,023 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName b 2024-12-01T14:30:14,023 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:14,023 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:14,023 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,024 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName c 2024-12-01T14:30:14,024 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:14,025 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:14,025 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,025 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,026 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,027 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,027 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,027 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:30:14,028 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,030 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T14:30:14,031 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bdc134bd4398a426a60f85f3d895c84; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74274533, jitterRate=0.10677678883075714}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:30:14,031 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bdc134bd4398a426a60f85f3d895c84: Writing region info on filesystem at 1733063414019Initializing all the Stores at 1733063414020 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063414020Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063414020Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063414020Cleaning up temporary data from old regions at 1733063414027 (+7 ms)Region opened successfully at 1733063414031 (+4 ms) 2024-12-01T14:30:14,031 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9bdc134bd4398a426a60f85f3d895c84, disabling compactions & flushes 2024-12-01T14:30:14,031 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:14,031 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:14,031 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. after waiting 0 ms 2024-12-01T14:30:14,031 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:14,031 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:14,031 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9bdc134bd4398a426a60f85f3d895c84: Waiting for close lock at 1733063414031Disabling compacts and flushes for region at 1733063414031Disabling writes for close at 1733063414031Writing region close event to WAL at 1733063414031Closed at 1733063414031 2024-12-01T14:30:14,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741925_1104 (size=95) 2024-12-01T14:30:14,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741925_1104 (size=95) 2024-12-01T14:30:14,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741925_1104 (size=95) 2024-12-01T14:30:14,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:30:14,035 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-35393853:(num 1733063413994) 2024-12-01T14:30:14,035 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:30:14,037 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:14,049 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037, exclude list is [], retry=0 2024-12-01T14:30:14,052 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:14,053 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:14,053 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:14,054 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037 2024-12-01T14:30:14,054 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:33969:33969),(127.0.0.1/127.0.0.1:36303:36303)] 2024-12-01T14:30:14,055 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bdc134bd4398a426a60f85f3d895c84, NAME => 'testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:30:14,055 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:14,055 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,055 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,056 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,057 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName a 2024-12-01T14:30:14,057 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:14,057 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:14,057 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,058 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName b 2024-12-01T14:30:14,058 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:14,058 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:14,059 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,059 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName c 2024-12-01T14:30:14,059 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:14,060 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:14,060 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,061 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,062 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,063 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,063 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,063 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:30:14,064 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,065 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bdc134bd4398a426a60f85f3d895c84; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72607836, jitterRate=0.08194106817245483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:30:14,065 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bdc134bd4398a426a60f85f3d895c84: Writing region info on filesystem at 1733063414055Initializing all the Stores at 1733063414056 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063414056Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063414056Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063414056Cleaning up temporary data from old regions at 1733063414063 (+7 ms)Region opened successfully at 1733063414065 (+2 ms) 2024-12-01T14:30:14,072 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9bdc134bd4398a426a60f85f3d895c84 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-01T14:30:14,088 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/a/024f8b16d15f457fa5b3c25cb3f673f2 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733063414065/Put/seqid=0 2024-12-01T14:30:14,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741928_1107 (size=5958) 2024-12-01T14:30:14,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741928_1107 (size=5958) 2024-12-01T14:30:14,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741928_1107 (size=5958) 2024-12-01T14:30:14,094 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/a/024f8b16d15f457fa5b3c25cb3f673f2 2024-12-01T14:30:14,099 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/a/024f8b16d15f457fa5b3c25cb3f673f2 as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/a/024f8b16d15f457fa5b3c25cb3f673f2 2024-12-01T14:30:14,105 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/a/024f8b16d15f457fa5b3c25cb3f673f2, entries=10, sequenceid=13, filesize=5.8 K 2024-12-01T14:30:14,106 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 9bdc134bd4398a426a60f85f3d895c84 in 34ms, sequenceid=13, compaction requested=false 2024-12-01T14:30:14,106 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9bdc134bd4398a426a60f85f3d895c84: 2024-12-01T14:30:14,125 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9bdc134bd4398a426a60f85f3d895c84, disabling compactions & flushes 2024-12-01T14:30:14,125 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:14,125 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:14,125 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. after waiting 0 ms 2024-12-01T14:30:14,125 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:14,126 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:14,126 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:14,126 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9bdc134bd4398a426a60f85f3d895c84: Waiting for close lock at 1733063414125Disabling compacts and flushes for region at 1733063414125Disabling writes for close at 1733063414125Writing region close event to WAL at 1733063414126 (+1 ms)Closed at 1733063414126 2024-12-01T14:30:14,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741927_1106 (size=3345) 2024-12-01T14:30:14,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741927_1106 (size=3345) 2024-12-01T14:30:14,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741927_1106 (size=3345) 2024-12-01T14:30:14,144 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037, size=3.3 K (3345bytes) 2024-12-01T14:30:14,144 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037 2024-12-01T14:30:14,144 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037 after 0ms 2024-12-01T14:30:14,146 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:14,146 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037 took 3ms 2024-12-01T14:30:14,148 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037 so closing down 2024-12-01T14:30:14,148 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:30:14,149 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733063414037.temp 2024-12-01T14:30:14,150 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000003-wal.1733063414037.temp 2024-12-01T14:30:14,150 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:30:14,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741929_1108 (size=2944) 2024-12-01T14:30:14,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741929_1108 (size=2944) 2024-12-01T14:30:14,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741929_1108 (size=2944) 2024-12-01T14:30:14,158 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000003-wal.1733063414037.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-01T14:30:14,159 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000003-wal.1733063414037.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000035 2024-12-01T14:30:14,160 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037, size=3.3 K, length=3345, corrupted=false, cancelled=false 2024-12-01T14:30:14,160 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037, journal: Splitting hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037, size=3.3 K (3345bytes) at 1733063414144Finishing writing output for hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037 so closing down at 1733063414148 (+4 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000003-wal.1733063414037.temp at 1733063414150 (+2 ms)3 split writer threads finished at 1733063414150Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000003-wal.1733063414037.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733063414158 (+8 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000003-wal.1733063414037.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000035 at 1733063414159 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037, size=3.3 K, length=3345, corrupted=false, cancelled=false at 1733063414160 (+1 ms) 2024-12-01T14:30:14,161 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414037 to hdfs://localhost:40889/hbase/oldWALs/wal.1733063414037 2024-12-01T14:30:14,162 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000035 2024-12-01T14:30:14,162 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:30:14,164 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:14,175 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164, exclude list is [], retry=0 2024-12-01T14:30:14,177 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:14,178 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:14,178 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:14,178 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T14:30:14,180 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 2024-12-01T14:30:14,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:30:14,180 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bdc134bd4398a426a60f85f3d895c84, NAME => 'testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84.', STARTKEY => '', ENDKEY => ''} 2024-12-01T14:30:14,180 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:14,180 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,180 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,182 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,183 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName a 2024-12-01T14:30:14,183 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:14,189 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/a/024f8b16d15f457fa5b3c25cb3f673f2 2024-12-01T14:30:14,189 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:14,189 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,190 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName b 2024-12-01T14:30:14,190 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:14,190 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:14,191 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,191 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName c 2024-12-01T14:30:14,191 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:14,192 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:14,192 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,193 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,194 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000035 2024-12-01T14:30:14,196 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:14,197 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000035 2024-12-01T14:30:14,197 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9bdc134bd4398a426a60f85f3d895c84 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-01T14:30:14,216 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/b/53f6abe445df473ebd51d957ae04ad6d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733063414106/Put/seqid=0 2024-12-01T14:30:14,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741931_1110 (size=5958) 2024-12-01T14:30:14,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741931_1110 (size=5958) 2024-12-01T14:30:14,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741931_1110 (size=5958) 2024-12-01T14:30:14,225 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/b/53f6abe445df473ebd51d957ae04ad6d 2024-12-01T14:30:14,245 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/c/81e7de9900164d40b6b13c788b371374 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733063414114/Put/seqid=0 2024-12-01T14:30:14,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741932_1111 (size=5958) 2024-12-01T14:30:14,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741932_1111 (size=5958) 2024-12-01T14:30:14,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741932_1111 (size=5958) 2024-12-01T14:30:14,251 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/c/81e7de9900164d40b6b13c788b371374 2024-12-01T14:30:14,258 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/b/53f6abe445df473ebd51d957ae04ad6d as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/b/53f6abe445df473ebd51d957ae04ad6d 2024-12-01T14:30:14,263 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/b/53f6abe445df473ebd51d957ae04ad6d, entries=10, sequenceid=35, filesize=5.8 K 2024-12-01T14:30:14,264 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/c/81e7de9900164d40b6b13c788b371374 as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/c/81e7de9900164d40b6b13c788b371374 2024-12-01T14:30:14,269 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/c/81e7de9900164d40b6b13c788b371374, entries=10, sequenceid=35, filesize=5.8 K 2024-12-01T14:30:14,269 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 9bdc134bd4398a426a60f85f3d895c84 in 72ms, sequenceid=35, compaction requested=false; wal=null 2024-12-01T14:30:14,270 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000035 2024-12-01T14:30:14,271 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,271 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,272 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:30:14,274 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:14,276 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-01T14:30:14,277 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bdc134bd4398a426a60f85f3d895c84; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61342636, jitterRate=-0.08592349290847778}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:30:14,277 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bdc134bd4398a426a60f85f3d895c84: Writing region info on filesystem at 1733063414180Initializing all the Stores at 1733063414181 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063414181Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063414182 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063414182Obtaining lock to block concurrent updates at 1733063414197 (+15 ms)Preparing flush snapshotting stores in 9bdc134bd4398a426a60f85f3d895c84 at 1733063414197Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733063414197Flushing stores of testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. at 1733063414197Flushing 9bdc134bd4398a426a60f85f3d895c84/b: creating writer at 1733063414197Flushing 9bdc134bd4398a426a60f85f3d895c84/b: appending metadata at 1733063414215 (+18 ms)Flushing 9bdc134bd4398a426a60f85f3d895c84/b: closing flushed file at 1733063414215Flushing 9bdc134bd4398a426a60f85f3d895c84/c: creating writer at 1733063414231 (+16 ms)Flushing 9bdc134bd4398a426a60f85f3d895c84/c: appending metadata at 1733063414244 (+13 ms)Flushing 9bdc134bd4398a426a60f85f3d895c84/c: closing flushed file at 1733063414244Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3aaacc80: reopening flushed file at 1733063414257 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59169bbf: reopening flushed file at 1733063414263 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 9bdc134bd4398a426a60f85f3d895c84 in 72ms, sequenceid=35, compaction requested=false; wal=null at 1733063414269 (+6 ms)Cleaning up temporary data from old regions at 1733063414271 (+2 ms)Region opened successfully at 1733063414277 (+6 ms) 2024-12-01T14:30:14,338 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164, size=0 (0bytes) 2024-12-01T14:30:14,338 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 might be still open, length is 0 2024-12-01T14:30:14,338 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 2024-12-01T14:30:14,338 WARN [IPC Server handler 2 on default port 40889 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-01T14:30:14,338 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 after 0ms 2024-12-01T14:30:15,490 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:57084 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33303:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57084 dst: /127.0.0.1:33303 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33303 remote=/127.0.0.1:57084]. Total timeout mills is 60000, 58812 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:15,492 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:52880 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33423:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52880 dst: /127.0.0.1:33423 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:15,493 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-726243590_22 at /127.0.0.1:43376 [Receiving block BP-1542046065-172.17.0.3-1733063375577:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40063:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43376 dst: /127.0.0.1:40063 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:15,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741930_1112 (size=2936) 2024-12-01T14:30:15,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741930_1112 (size=2936) 2024-12-01T14:30:18,339 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 after 4001ms 2024-12-01T14:30:18,344 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:18,344 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 took 4007ms 2024-12-01T14:30:18,348 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164; continuing. 2024-12-01T14:30:18,348 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 so closing down 2024-12-01T14:30:18,348 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-01T14:30:18,351 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733063414164.temp 2024-12-01T14:30:18,352 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000037-wal.1733063414164.temp 2024-12-01T14:30:18,376 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-01T14:30:18,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741933_1113 (size=2944) 2024-12-01T14:30:18,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741933_1113 (size=2944) 2024-12-01T14:30:18,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741933_1113 (size=2944) 2024-12-01T14:30:18,386 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000037-wal.1733063414164.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-01T14:30:18,387 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000037-wal.1733063414164.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000066 2024-12-01T14:30:18,387 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164, size=0, length=0, corrupted=false, cancelled=false 2024-12-01T14:30:18,388 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164, journal: Splitting hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164, size=0 (0bytes) at 1733063414338Finishing writing output for hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 so closing down at 1733063418348 (+4010 ms)Creating recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000037-wal.1733063414164.temp at 1733063418375 (+27 ms)3 split writer threads finished at 1733063418376 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000037-wal.1733063414164.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733063418386 (+10 ms)Rename recovered edits hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000037-wal.1733063414164.temp to hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000066 at 1733063418387 (+1 ms)Processed 30 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164, size=0, length=0, corrupted=false, cancelled=false at 1733063418388 (+1 ms) 2024-12-01T14:30:18,389 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 to hdfs://localhost:40889/hbase/oldWALs/wal.1733063414164 2024-12-01T14:30:18,390 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000066 2024-12-01T14:30:18,390 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-01T14:30:18,391 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40889/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988, archiveDir=hdfs://localhost:40889/hbase/oldWALs, maxLogs=32 2024-12-01T14:30:18,403 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063418392, exclude list is [], retry=0 2024-12-01T14:30:18,406 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33303,DS-0a95010c-7d31-41b3-acf3-b49ac3735c69,DISK] 2024-12-01T14:30:18,406 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33423,DS-839bb17d-2244-4d81-b1ed-c0592745fdb4,DISK] 2024-12-01T14:30:18,406 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40063,DS-047d8189-f10b-4007-89ba-d48b429cb5d6,DISK] 2024-12-01T14:30:18,408 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063418392 2024-12-01T14:30:18,408 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40289:40289),(127.0.0.1/127.0.0.1:36303:36303),(127.0.0.1/127.0.0.1:33969:33969)] 2024-12-01T14:30:18,408 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T14:30:18,410 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:18,410 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName a 2024-12-01T14:30:18,410 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:18,415 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/a/024f8b16d15f457fa5b3c25cb3f673f2 2024-12-01T14:30:18,415 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:18,415 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:18,415 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName b 2024-12-01T14:30:18,416 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:18,420 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/b/53f6abe445df473ebd51d957ae04ad6d 2024-12-01T14:30:18,420 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:18,420 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:18,421 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bdc134bd4398a426a60f85f3d895c84 columnFamilyName c 2024-12-01T14:30:18,421 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T14:30:18,426 DEBUG [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/c/81e7de9900164d40b6b13c788b371374 2024-12-01T14:30:18,426 INFO [StoreOpener-9bdc134bd4398a426a60f85f3d895c84-1 {}] regionserver.HStore(327): Store=9bdc134bd4398a426a60f85f3d895c84/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T14:30:18,426 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:18,427 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:18,428 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:18,429 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000066 2024-12-01T14:30:18,430 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-01T14:30:18,434 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000066 2024-12-01T14:30:18,434 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9bdc134bd4398a426a60f85f3d895c84 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-01T14:30:18,447 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/a/216c070deb994391b5cdf37423c9e222 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733063414284/Put/seqid=0 2024-12-01T14:30:18,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741935_1115 (size=5958) 2024-12-01T14:30:18,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741935_1115 (size=5958) 2024-12-01T14:30:18,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741935_1115 (size=5958) 2024-12-01T14:30:18,454 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/a/216c070deb994391b5cdf37423c9e222 2024-12-01T14:30:18,473 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/b/a2e885615e5e46faa33d3a1b9e38882a is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733063414290/Put/seqid=0 2024-12-01T14:30:18,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741936_1116 (size=5958) 2024-12-01T14:30:18,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741936_1116 (size=5958) 2024-12-01T14:30:18,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741936_1116 (size=5958) 2024-12-01T14:30:18,480 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/b/a2e885615e5e46faa33d3a1b9e38882a 2024-12-01T14:30:18,502 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/c/9d7dd8aa833d4f1897084472e83395bf is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733063414297/Put/seqid=0 2024-12-01T14:30:18,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741937_1117 (size=5958) 2024-12-01T14:30:18,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741937_1117 (size=5958) 2024-12-01T14:30:18,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741937_1117 (size=5958) 2024-12-01T14:30:18,509 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/c/9d7dd8aa833d4f1897084472e83395bf 2024-12-01T14:30:18,514 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/a/216c070deb994391b5cdf37423c9e222 as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/a/216c070deb994391b5cdf37423c9e222 2024-12-01T14:30:18,518 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/a/216c070deb994391b5cdf37423c9e222, entries=10, sequenceid=66, filesize=5.8 K 2024-12-01T14:30:18,519 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/b/a2e885615e5e46faa33d3a1b9e38882a as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/b/a2e885615e5e46faa33d3a1b9e38882a 2024-12-01T14:30:18,525 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/b/a2e885615e5e46faa33d3a1b9e38882a, entries=10, sequenceid=66, filesize=5.8 K 2024-12-01T14:30:18,526 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/.tmp/c/9d7dd8aa833d4f1897084472e83395bf as hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/c/9d7dd8aa833d4f1897084472e83395bf 2024-12-01T14:30:18,530 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/c/9d7dd8aa833d4f1897084472e83395bf, entries=10, sequenceid=66, filesize=5.8 K 2024-12-01T14:30:18,531 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 9bdc134bd4398a426a60f85f3d895c84 in 97ms, sequenceid=66, compaction requested=false; wal=null 2024-12-01T14:30:18,531 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/0000000000000000066 2024-12-01T14:30:18,532 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:18,532 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:18,533 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-01T14:30:18,534 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9bdc134bd4398a426a60f85f3d895c84 2024-12-01T14:30:18,536 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/hbase/data/default/testReplayEditsWrittenViaHRegion/9bdc134bd4398a426a60f85f3d895c84/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-01T14:30:18,537 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9bdc134bd4398a426a60f85f3d895c84; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74148955, jitterRate=0.10490553081035614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-01T14:30:18,537 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9bdc134bd4398a426a60f85f3d895c84: Writing region info on filesystem at 1733063418408Initializing all the Stores at 1733063418409 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063418409Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063418409Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733063418409Obtaining lock to block concurrent updates at 1733063418434 (+25 ms)Preparing flush snapshotting stores in 9bdc134bd4398a426a60f85f3d895c84 at 1733063418434Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733063418434Flushing stores of testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. at 1733063418434Flushing 9bdc134bd4398a426a60f85f3d895c84/a: creating writer at 1733063418434Flushing 9bdc134bd4398a426a60f85f3d895c84/a: appending metadata at 1733063418447 (+13 ms)Flushing 9bdc134bd4398a426a60f85f3d895c84/a: closing flushed file at 1733063418447Flushing 9bdc134bd4398a426a60f85f3d895c84/b: creating writer at 1733063418458 (+11 ms)Flushing 9bdc134bd4398a426a60f85f3d895c84/b: appending metadata at 1733063418472 (+14 ms)Flushing 9bdc134bd4398a426a60f85f3d895c84/b: closing flushed file at 1733063418472Flushing 9bdc134bd4398a426a60f85f3d895c84/c: creating writer at 1733063418484 (+12 ms)Flushing 9bdc134bd4398a426a60f85f3d895c84/c: appending metadata at 1733063418502 (+18 ms)Flushing 9bdc134bd4398a426a60f85f3d895c84/c: closing flushed file at 1733063418502Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c3f9d15: reopening flushed file at 1733063418513 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50c64663: reopening flushed file at 1733063418519 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5adca800: reopening flushed file at 1733063418525 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 9bdc134bd4398a426a60f85f3d895c84 in 97ms, sequenceid=66, compaction requested=false; wal=null at 1733063418531 (+6 ms)Cleaning up temporary data from old regions at 1733063418532 (+1 ms)Region opened successfully at 1733063418537 (+5 ms) 2024-12-01T14:30:18,550 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9bdc134bd4398a426a60f85f3d895c84, disabling compactions & flushes 2024-12-01T14:30:18,550 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:18,550 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:18,550 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. after waiting 0 ms 2024-12-01T14:30:18,550 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:18,552 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733063413989.9bdc134bd4398a426a60f85f3d895c84. 2024-12-01T14:30:18,552 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9bdc134bd4398a426a60f85f3d895c84: Waiting for close lock at 1733063418550Disabling compacts and flushes for region at 1733063418550Disabling writes for close at 1733063418550Writing region close event to WAL at 1733063418552 (+2 ms)Closed at 1733063418552 2024-12-01T14:30:18,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741934_1114 (size=95) 2024-12-01T14:30:18,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741934_1114 (size=95) 2024-12-01T14:30:18,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741934_1114 (size=95) 2024-12-01T14:30:18,557 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-01T14:30:18,557 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733063418392) 2024-12-01T14:30:18,569 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=453 (was 445) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1064788188_22 at /127.0.0.1:52920 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1064788188_22 at /127.0.0.1:43428 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:40889 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1064788188_22 at /127.0.0.1:57160 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:40889 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1416 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=149 (was 162), ProcessCount=11 (was 11), AvailableMemoryMB=4824 (was 4865) 2024-12-01T14:30:18,570 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1416 is superior to 1024 2024-12-01T14:30:18,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-01T14:30:18,570 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T14:30:18,570 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:30:18,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:30:18,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:30:18,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T14:30:18,571 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-01T14:30:18,571 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1265140316, stopped=false 2024-12-01T14:30:18,571 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=57b748500ef9,34327,1733063380396 2024-12-01T14:30:18,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T14:30:18,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T14:30:18,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T14:30:18,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:30:18,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:30:18,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:30:18,637 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T14:30:18,639 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:30:18,639 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T14:30:18,639 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:30:18,640 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:30:18,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:30:18,640 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '57b748500ef9,38129,1733063381264' ***** 2024-12-01T14:30:18,640 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T14:30:18,640 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '57b748500ef9,40449,1733063381307' ***** 2024-12-01T14:30:18,640 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T14:30:18,640 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T14:30:18,641 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T14:30:18,641 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T14:30:18,641 INFO [RS:2;57b748500ef9:40449 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T14:30:18,641 INFO [RS:1;57b748500ef9:38129 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T14:30:18,641 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T14:30:18,641 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T14:30:18,641 INFO [RS:1;57b748500ef9:38129 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T14:30:18,641 INFO [RS:2;57b748500ef9:40449 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T14:30:18,642 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(959): stopping server 57b748500ef9,38129,1733063381264 2024-12-01T14:30:18,642 INFO [RS:1;57b748500ef9:38129 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:30:18,642 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(3091): Received CLOSE for 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:30:18,642 INFO [RS:1;57b748500ef9:38129 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;57b748500ef9:38129. 2024-12-01T14:30:18,642 DEBUG [RS:1;57b748500ef9:38129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:30:18,642 DEBUG [RS:1;57b748500ef9:38129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:30:18,642 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(959): stopping server 57b748500ef9,40449,1733063381307 2024-12-01T14:30:18,642 INFO [RS:1;57b748500ef9:38129 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T14:30:18,642 INFO [RS:2;57b748500ef9:40449 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:30:18,642 INFO [RS:1;57b748500ef9:38129 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T14:30:18,642 INFO [RS:2;57b748500ef9:40449 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;57b748500ef9:40449. 2024-12-01T14:30:18,642 INFO [RS:1;57b748500ef9:38129 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T14:30:18,642 DEBUG [RS:2;57b748500ef9:40449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T14:30:18,642 DEBUG [RS:2;57b748500ef9:40449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:30:18,642 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T14:30:18,642 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-01T14:30:18,642 DEBUG [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(1325): Online Regions={44c73b6c4a612bcfc6cf8a6997a1cd8e=testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e.} 2024-12-01T14:30:18,642 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 44c73b6c4a612bcfc6cf8a6997a1cd8e, disabling compactions & flushes 2024-12-01T14:30:18,643 DEBUG [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(1351): Waiting on 44c73b6c4a612bcfc6cf8a6997a1cd8e 2024-12-01T14:30:18,643 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:30:18,643 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:30:18,643 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. after waiting 0 ms 2024-12-01T14:30:18,643 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:30:18,643 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-01T14:30:18,643 DEBUG [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-01T14:30:18,643 DEBUG [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-01T14:30:18,643 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T14:30:18,643 INFO [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T14:30:18,643 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T14:30:18,643 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T14:30:18,643 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T14:30:18,643 INFO [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-01T14:30:18,647 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/default/testReplayEditsAfterRegionMovedWithMultiCF/44c73b6c4a612bcfc6cf8a6997a1cd8e/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-01T14:30:18,648 INFO [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:30:18,648 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 44c73b6c4a612bcfc6cf8a6997a1cd8e: Waiting for close lock at 1733063418642Running coprocessor pre-close hooks at 1733063418642Disabling compacts and flushes for region at 1733063418642Disabling writes for close at 1733063418643 (+1 ms)Writing region close event to WAL at 1733063418644 (+1 ms)Running coprocessor post-close hooks at 1733063418648 (+4 ms)Closed at 1733063418648 2024-12-01T14:30:18,648 DEBUG [RS_CLOSE_REGION-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e. 2024-12-01T14:30:18,661 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/.tmp/info/5375be130a0b4a7cb0f96a61ac2cf455 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733063395533.44c73b6c4a612bcfc6cf8a6997a1cd8e./info:regioninfo/1733063398690/Put/seqid=0 2024-12-01T14:30:18,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741938_1118 (size=8243) 2024-12-01T14:30:18,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741938_1118 (size=8243) 2024-12-01T14:30:18,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741938_1118 (size=8243) 2024-12-01T14:30:18,667 INFO [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/.tmp/info/5375be130a0b4a7cb0f96a61ac2cf455 2024-12-01T14:30:18,676 INFO [regionserver/57b748500ef9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:30:18,676 INFO [regionserver/57b748500ef9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:30:18,686 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/.tmp/ns/d8946c41c0d946f89fe0e943e2525f0d is 43, key is default/ns:d/1733063383428/Put/seqid=0 2024-12-01T14:30:18,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741939_1119 (size=5153) 2024-12-01T14:30:18,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741939_1119 (size=5153) 2024-12-01T14:30:18,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741939_1119 (size=5153) 2024-12-01T14:30:18,692 INFO [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/.tmp/ns/d8946c41c0d946f89fe0e943e2525f0d 2024-12-01T14:30:18,714 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/.tmp/table/dbf72507e74c4a0db36039f6a6844fdd is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733063395970/Put/seqid=0 2024-12-01T14:30:18,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741940_1120 (size=5431) 2024-12-01T14:30:18,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741940_1120 (size=5431) 2024-12-01T14:30:18,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741940_1120 (size=5431) 2024-12-01T14:30:18,720 INFO [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/.tmp/table/dbf72507e74c4a0db36039f6a6844fdd 2024-12-01T14:30:18,725 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/.tmp/info/5375be130a0b4a7cb0f96a61ac2cf455 as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/info/5375be130a0b4a7cb0f96a61ac2cf455 2024-12-01T14:30:18,730 INFO [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/info/5375be130a0b4a7cb0f96a61ac2cf455, entries=18, sequenceid=21, filesize=8.0 K 2024-12-01T14:30:18,731 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/.tmp/ns/d8946c41c0d946f89fe0e943e2525f0d as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/ns/d8946c41c0d946f89fe0e943e2525f0d 2024-12-01T14:30:18,736 INFO [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/ns/d8946c41c0d946f89fe0e943e2525f0d, entries=2, sequenceid=21, filesize=5.0 K 2024-12-01T14:30:18,737 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/.tmp/table/dbf72507e74c4a0db36039f6a6844fdd as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/table/dbf72507e74c4a0db36039f6a6844fdd 2024-12-01T14:30:18,743 INFO [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/table/dbf72507e74c4a0db36039f6a6844fdd, entries=2, sequenceid=21, filesize=5.3 K 2024-12-01T14:30:18,744 INFO [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=21, compaction requested=false 2024-12-01T14:30:18,750 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-01T14:30:18,751 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T14:30:18,751 INFO [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T14:30:18,751 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733063418643Running coprocessor pre-close hooks at 1733063418643Disabling compacts and flushes for region at 1733063418643Disabling writes for close at 1733063418643Obtaining lock to block concurrent updates at 1733063418643Preparing flush snapshotting stores in 1588230740 at 1733063418643Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733063418644 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733063418645 (+1 ms)Flushing 1588230740/info: creating writer at 1733063418645Flushing 1588230740/info: appending metadata at 1733063418661 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733063418661Flushing 1588230740/ns: creating writer at 1733063418672 (+11 ms)Flushing 1588230740/ns: appending metadata at 1733063418686 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733063418686Flushing 1588230740/table: creating writer at 1733063418697 (+11 ms)Flushing 1588230740/table: appending metadata at 1733063418713 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733063418713Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34ec77fb: reopening flushed file at 1733063418724 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3195efcc: reopening flushed file at 1733063418731 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a55f0b0: reopening flushed file at 1733063418737 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=21, compaction requested=false at 1733063418744 (+7 ms)Writing region close event to WAL at 1733063418746 (+2 ms)Running coprocessor post-close hooks at 1733063418750 (+4 ms)Closed at 1733063418751 (+1 ms) 2024-12-01T14:30:18,751 DEBUG [RS_CLOSE_META-regionserver/57b748500ef9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T14:30:18,843 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(976): stopping server 57b748500ef9,40449,1733063381307; all regions closed. 2024-12-01T14:30:18,843 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(976): stopping server 57b748500ef9,38129,1733063381264; all regions closed. 2024-12-01T14:30:18,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741833_1009 (size=720) 2024-12-01T14:30:18,850 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,40449,1733063381307/57b748500ef9%2C40449%2C1733063381307.1733063382781 not finished, retry = 0 2024-12-01T14:30:18,850 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/WALs/57b748500ef9,38129,1733063381264/57b748500ef9%2C38129%2C1733063381264.meta.1733063383267.meta not finished, retry = 0 2024-12-01T14:30:18,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741836_1012 (size=4708) 2024-12-01T14:30:18,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741833_1009 (size=720) 2024-12-01T14:30:18,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741833_1009 (size=720) 2024-12-01T14:30:18,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741836_1012 (size=4708) 2024-12-01T14:30:18,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741836_1012 (size=4708) 2024-12-01T14:30:18,956 DEBUG [RS:2;57b748500ef9:40449 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/oldWALs 2024-12-01T14:30:18,956 INFO [RS:2;57b748500ef9:40449 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 57b748500ef9%2C40449%2C1733063381307:(num 1733063382781) 2024-12-01T14:30:18,956 DEBUG [RS:2;57b748500ef9:40449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:30:18,956 DEBUG [RS:1;57b748500ef9:38129 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/oldWALs 2024-12-01T14:30:18,956 INFO [RS:2;57b748500ef9:40449 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:30:18,956 INFO [RS:1;57b748500ef9:38129 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 57b748500ef9%2C38129%2C1733063381264.meta:.meta(num 1733063383267) 2024-12-01T14:30:18,956 INFO [RS:2;57b748500ef9:40449 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:30:18,956 INFO [RS:2;57b748500ef9:40449 {}] hbase.ChoreService(370): Chore service for: regionserver/57b748500ef9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-01T14:30:18,957 INFO [RS:2;57b748500ef9:40449 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T14:30:18,957 INFO [RS:2;57b748500ef9:40449 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T14:30:18,957 INFO [regionserver/57b748500ef9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:30:18,957 INFO [RS:2;57b748500ef9:40449 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T14:30:18,957 INFO [RS:2;57b748500ef9:40449 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:30:18,957 INFO [RS:2;57b748500ef9:40449 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40449 2024-12-01T14:30:18,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741835_1011 (size=1707) 2024-12-01T14:30:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741835_1011 (size=1707) 2024-12-01T14:30:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741835_1011 (size=1707) 2024-12-01T14:30:18,964 DEBUG [RS:1;57b748500ef9:38129 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/oldWALs 2024-12-01T14:30:18,964 INFO [RS:1;57b748500ef9:38129 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 57b748500ef9%2C38129%2C1733063381264:(num 1733063382781) 2024-12-01T14:30:18,964 DEBUG [RS:1;57b748500ef9:38129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T14:30:18,964 INFO [RS:1;57b748500ef9:38129 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T14:30:18,964 INFO [RS:1;57b748500ef9:38129 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:30:18,964 INFO [RS:1;57b748500ef9:38129 {}] hbase.ChoreService(370): Chore service for: regionserver/57b748500ef9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-01T14:30:18,964 INFO [RS:1;57b748500ef9:38129 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:30:18,964 INFO [regionserver/57b748500ef9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:30:18,965 INFO [RS:1;57b748500ef9:38129 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38129 2024-12-01T14:30:18,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/57b748500ef9,40449,1733063381307 2024-12-01T14:30:18,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T14:30:18,978 INFO [RS:2;57b748500ef9:40449 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:30:18,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/57b748500ef9,38129,1733063381264 2024-12-01T14:30:18,988 INFO [RS:1;57b748500ef9:38129 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:30:18,999 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [57b748500ef9,38129,1733063381264] 2024-12-01T14:30:19,020 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/57b748500ef9,38129,1733063381264 already deleted, retry=false 2024-12-01T14:30:19,020 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 57b748500ef9,38129,1733063381264 expired; onlineServers=1 2024-12-01T14:30:19,020 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [57b748500ef9,40449,1733063381307] 2024-12-01T14:30:19,030 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/57b748500ef9,40449,1733063381307 already deleted, retry=false 2024-12-01T14:30:19,030 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 57b748500ef9,40449,1733063381307 expired; onlineServers=0 2024-12-01T14:30:19,030 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '57b748500ef9,34327,1733063380396' ***** 2024-12-01T14:30:19,031 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-01T14:30:19,031 INFO [M:0;57b748500ef9:34327 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T14:30:19,031 INFO [M:0;57b748500ef9:34327 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T14:30:19,031 DEBUG [M:0;57b748500ef9:34327 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-01T14:30:19,031 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-01T14:30:19,032 DEBUG [M:0;57b748500ef9:34327 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-01T14:30:19,032 DEBUG [master/57b748500ef9:0:becomeActiveMaster-HFileCleaner.small.0-1733063382438 {}] cleaner.HFileCleaner(306): Exit Thread[master/57b748500ef9:0:becomeActiveMaster-HFileCleaner.small.0-1733063382438,5,FailOnTimeoutGroup] 2024-12-01T14:30:19,032 DEBUG [master/57b748500ef9:0:becomeActiveMaster-HFileCleaner.large.0-1733063382438 {}] cleaner.HFileCleaner(306): Exit Thread[master/57b748500ef9:0:becomeActiveMaster-HFileCleaner.large.0-1733063382438,5,FailOnTimeoutGroup] 2024-12-01T14:30:19,032 INFO [M:0;57b748500ef9:34327 {}] hbase.ChoreService(370): Chore service for: master/57b748500ef9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-01T14:30:19,033 INFO [M:0;57b748500ef9:34327 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T14:30:19,033 DEBUG [M:0;57b748500ef9:34327 {}] master.HMaster(1795): Stopping service threads 2024-12-01T14:30:19,033 INFO [M:0;57b748500ef9:34327 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-01T14:30:19,033 INFO [M:0;57b748500ef9:34327 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T14:30:19,034 INFO [M:0;57b748500ef9:34327 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-01T14:30:19,034 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-01T14:30:19,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-01T14:30:19,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T14:30:19,041 DEBUG [M:0;57b748500ef9:34327 {}] zookeeper.ZKUtil(347): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-01T14:30:19,042 WARN [M:0;57b748500ef9:34327 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-01T14:30:19,043 INFO [M:0;57b748500ef9:34327 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/.lastflushedseqids 2024-12-01T14:30:19,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741941_1121 (size=138) 2024-12-01T14:30:19,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741941_1121 (size=138) 2024-12-01T14:30:19,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741941_1121 (size=138) 2024-12-01T14:30:19,059 INFO [M:0;57b748500ef9:34327 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-01T14:30:19,060 INFO [M:0;57b748500ef9:34327 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-01T14:30:19,060 DEBUG [M:0;57b748500ef9:34327 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T14:30:19,060 INFO [M:0;57b748500ef9:34327 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:30:19,060 DEBUG [M:0;57b748500ef9:34327 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:30:19,060 DEBUG [M:0;57b748500ef9:34327 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T14:30:19,060 DEBUG [M:0;57b748500ef9:34327 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:30:19,060 INFO [M:0;57b748500ef9:34327 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-12-01T14:30:19,074 DEBUG [M:0;57b748500ef9:34327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e6f79d2f5c2647be8cec6c3bc8f8b191 is 82, key is hbase:meta,,1/info:regioninfo/1733063383345/Put/seqid=0 2024-12-01T14:30:19,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741942_1122 (size=5672) 2024-12-01T14:30:19,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741942_1122 (size=5672) 2024-12-01T14:30:19,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741942_1122 (size=5672) 2024-12-01T14:30:19,081 INFO [M:0;57b748500ef9:34327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e6f79d2f5c2647be8cec6c3bc8f8b191 2024-12-01T14:30:19,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:30:19,099 INFO [RS:2;57b748500ef9:40449 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:30:19,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40449-0x101914c865a0003, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:30:19,099 INFO [RS:2;57b748500ef9:40449 {}] regionserver.HRegionServer(1031): Exiting; stopping=57b748500ef9,40449,1733063381307; zookeeper connection closed. 2024-12-01T14:30:19,099 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a38245f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a38245f 2024-12-01T14:30:19,100 DEBUG [M:0;57b748500ef9:34327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7bfb6235e7f442b8d58d886297e8321 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733063395976/Put/seqid=0 2024-12-01T14:30:19,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741943_1123 (size=7755) 2024-12-01T14:30:19,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741943_1123 (size=7755) 2024-12-01T14:30:19,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741943_1123 (size=7755) 2024-12-01T14:30:19,106 INFO [M:0;57b748500ef9:34327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7bfb6235e7f442b8d58d886297e8321 2024-12-01T14:30:19,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:30:19,110 INFO [RS:1;57b748500ef9:38129 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:30:19,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x101914c865a0002, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:30:19,110 INFO [RS:1;57b748500ef9:38129 {}] regionserver.HRegionServer(1031): Exiting; stopping=57b748500ef9,38129,1733063381264; zookeeper connection closed. 2024-12-01T14:30:19,110 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2297204e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2297204e 2024-12-01T14:30:19,110 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-01T14:30:19,111 INFO [M:0;57b748500ef9:34327 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7bfb6235e7f442b8d58d886297e8321 2024-12-01T14:30:19,130 DEBUG [M:0;57b748500ef9:34327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/541a6baa48ef4c5281d7cf8d5d3055d2 is 69, key is 57b748500ef9,38129,1733063381264/rs:state/1733063382491/Put/seqid=0 2024-12-01T14:30:19,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741944_1124 (size=5440) 2024-12-01T14:30:19,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741944_1124 (size=5440) 2024-12-01T14:30:19,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741944_1124 (size=5440) 2024-12-01T14:30:19,137 INFO [M:0;57b748500ef9:34327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/541a6baa48ef4c5281d7cf8d5d3055d2 2024-12-01T14:30:19,141 INFO [M:0;57b748500ef9:34327 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 541a6baa48ef4c5281d7cf8d5d3055d2 2024-12-01T14:30:19,142 DEBUG [M:0;57b748500ef9:34327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e6f79d2f5c2647be8cec6c3bc8f8b191 as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e6f79d2f5c2647be8cec6c3bc8f8b191 2024-12-01T14:30:19,146 INFO [M:0;57b748500ef9:34327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e6f79d2f5c2647be8cec6c3bc8f8b191, entries=8, sequenceid=168, filesize=5.5 K 2024-12-01T14:30:19,147 DEBUG [M:0;57b748500ef9:34327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c7bfb6235e7f442b8d58d886297e8321 as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c7bfb6235e7f442b8d58d886297e8321 2024-12-01T14:30:19,151 INFO [M:0;57b748500ef9:34327 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7bfb6235e7f442b8d58d886297e8321 2024-12-01T14:30:19,151 INFO [M:0;57b748500ef9:34327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c7bfb6235e7f442b8d58d886297e8321, entries=17, sequenceid=168, filesize=7.6 K 2024-12-01T14:30:19,152 DEBUG [M:0;57b748500ef9:34327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/541a6baa48ef4c5281d7cf8d5d3055d2 as hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/541a6baa48ef4c5281d7cf8d5d3055d2 2024-12-01T14:30:19,157 INFO [M:0;57b748500ef9:34327 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 541a6baa48ef4c5281d7cf8d5d3055d2 2024-12-01T14:30:19,157 INFO [M:0;57b748500ef9:34327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40889/user/jenkins/test-data/95ec8721-e57a-2178-e88a-fa7ede0d6aa9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/541a6baa48ef4c5281d7cf8d5d3055d2, entries=3, sequenceid=168, filesize=5.3 K 2024-12-01T14:30:19,158 INFO [M:0;57b748500ef9:34327 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 98ms, sequenceid=168, compaction requested=false 2024-12-01T14:30:19,159 INFO [M:0;57b748500ef9:34327 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T14:30:19,159 DEBUG [M:0;57b748500ef9:34327 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733063419060Disabling compacts and flushes for region at 1733063419060Disabling writes for close at 1733063419060Obtaining lock to block concurrent updates at 1733063419060Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733063419060Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1733063419060Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733063419061 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733063419061Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733063419074 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733063419074Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733063419085 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733063419099 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733063419099Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733063419111 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733063419129 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733063419129Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a3b29e8: reopening flushed file at 1733063419141 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1109ca60: reopening flushed file at 1733063419146 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f3935b1: reopening flushed file at 1733063419151 (+5 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 98ms, sequenceid=168, compaction requested=false at 1733063419158 (+7 ms)Writing region close event to WAL at 1733063419159 (+1 ms)Closed at 1733063419159 2024-12-01T14:30:19,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40063 is added to blk_1073741830_1006 (size=56548) 2024-12-01T14:30:19,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33303 is added to blk_1073741830_1006 (size=56548) 2024-12-01T14:30:19,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33423 is added to blk_1073741830_1006 (size=56548) 2024-12-01T14:30:19,162 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T14:30:19,163 INFO [M:0;57b748500ef9:34327 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-01T14:30:19,163 INFO [M:0;57b748500ef9:34327 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34327 2024-12-01T14:30:19,163 INFO [M:0;57b748500ef9:34327 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T14:30:19,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:30:19,273 INFO [M:0;57b748500ef9:34327 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T14:30:19,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34327-0x101914c865a0000, quorum=127.0.0.1:56293, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T14:30:19,279 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399314 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399314 (inode 16655) Holder DFSClient_NONMAPREDUCE_-726243590_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733063398999/wal.1733063399314 (inode 16655) Holder DFSClient_NONMAPREDUCE_-726243590_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-01T14:30:19,280 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733063391024/wal.1733063391254 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:19,280 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733063413988/wal.1733063414164 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:19,282 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063408705 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063408705 (inode 16678) Holder DFSClient_NONMAPREDUCE_-726243590_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733063399451/wal.1733063408705 (inode 16678) Holder DFSClient_NONMAPREDUCE_-726243590_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-01T14:30:19,283 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733063384431/wal.1733063384499 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:19,285 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063409384 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063409384 (inode 16704) Holder DFSClient_NONMAPREDUCE_-726243590_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733063408953/wal.1733063409384 (inode 16704) Holder DFSClient_NONMAPREDUCE_-726243590_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-01T14:30:19,286 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733063409527/wal.1733063409583 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T14:30:19,288 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal.1733063384306 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal.1733063384306 (inode 16485) Holder DFSClient_NONMAPREDUCE_-726243590_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733063384106/wal.1733063384306 (inode 16485) Holder DFSClient_NONMAPREDUCE_-726243590_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-01T14:30:19,290 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal.1733063383874 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal.1733063383874 (inode 16462) Holder DFSClient_NONMAPREDUCE_-726243590_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733063383630/wal.1733063383874 (inode 16462) Holder DFSClient_NONMAPREDUCE_-726243590_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-01T14:30:19,295 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T14:30:19,297 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T14:30:19,297 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T14:30:19,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T14:30:19,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/hadoop.log.dir/,STOPPED} 2024-12-01T14:30:19,301 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T14:30:19,301 WARN [BP-1542046065-172.17.0.3-1733063375577 heartbeating to localhost/127.0.0.1:40889 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T14:30:19,301 WARN [BP-1542046065-172.17.0.3-1733063375577 heartbeating to localhost/127.0.0.1:40889 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1542046065-172.17.0.3-1733063375577 (Datanode Uuid 07777e45-b85e-40f3-b473-ca12a96b8269) service to localhost/127.0.0.1:40889 2024-12-01T14:30:19,301 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T14:30:19,302 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data5/current/BP-1542046065-172.17.0.3-1733063375577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T14:30:19,302 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data6/current/BP-1542046065-172.17.0.3-1733063375577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T14:30:19,303 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T14:30:19,305 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T14:30:19,305 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T14:30:19,305 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T14:30:19,305 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T14:30:19,305 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/hadoop.log.dir/,STOPPED} 2024-12-01T14:30:19,306 WARN [BP-1542046065-172.17.0.3-1733063375577 heartbeating to localhost/127.0.0.1:40889 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T14:30:19,306 WARN [BP-1542046065-172.17.0.3-1733063375577 heartbeating to localhost/127.0.0.1:40889 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1542046065-172.17.0.3-1733063375577 (Datanode Uuid 83924cbb-93f5-4467-bf71-2c95bfcc03dc) service to localhost/127.0.0.1:40889 2024-12-01T14:30:19,307 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data3/current/BP-1542046065-172.17.0.3-1733063375577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T14:30:19,307 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data4/current/BP-1542046065-172.17.0.3-1733063375577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T14:30:19,307 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T14:30:19,307 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T14:30:19,307 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T14:30:19,309 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T14:30:19,309 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T14:30:19,309 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T14:30:19,309 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T14:30:19,310 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/hadoop.log.dir/,STOPPED} 2024-12-01T14:30:19,312 WARN [BP-1542046065-172.17.0.3-1733063375577 heartbeating to localhost/127.0.0.1:40889 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T14:30:19,312 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T14:30:19,312 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T14:30:19,312 WARN [BP-1542046065-172.17.0.3-1733063375577 heartbeating to localhost/127.0.0.1:40889 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1542046065-172.17.0.3-1733063375577 (Datanode Uuid b8de6bcc-2406-43e2-9e37-8edacfbb4cc0) service to localhost/127.0.0.1:40889 2024-12-01T14:30:19,313 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data1/current/BP-1542046065-172.17.0.3-1733063375577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T14:30:19,313 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/cluster_7b09d697-a563-2163-4a5a-cb82f2d8d233/data/data2/current/BP-1542046065-172.17.0.3-1733063375577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T14:30:19,314 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T14:30:19,319 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-01T14:30:19,320 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T14:30:19,320 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T14:30:19,320 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T14:30:19,321 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/772d452f-2e1a-f1c8-edef-fc96b63b2726/hadoop.log.dir/,STOPPED} 2024-12-01T14:30:19,329 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-01T14:30:19,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down