2024-11-26 23:25:11,451 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-26 23:25:11,462 main DEBUG Took 0.009527 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 23:25:11,463 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 23:25:11,463 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 23:25:11,464 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 23:25:11,465 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,472 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 23:25:11,483 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,485 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,486 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,486 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,486 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,487 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,488 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,488 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,488 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,489 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,489 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,490 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,490 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,491 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,491 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,491 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,492 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,492 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,493 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,493 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,493 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,494 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:25:11,494 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,494 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 23:25:11,496 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:25:11,497 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 23:25:11,499 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 23:25:11,499 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 23:25:11,500 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 23:25:11,500 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 23:25:11,509 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 23:25:11,511 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 23:25:11,513 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 23:25:11,513 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 23:25:11,514 main DEBUG createAppenders(={Console}) 2024-11-26 23:25:11,514 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-26 23:25:11,515 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-26 23:25:11,515 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-26 23:25:11,515 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 23:25:11,516 main DEBUG OutputStream closed 2024-11-26 23:25:11,516 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 23:25:11,516 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 23:25:11,516 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-26 23:25:11,582 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 23:25:11,584 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 23:25:11,585 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 23:25:11,586 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 23:25:11,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 23:25:11,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 23:25:11,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 23:25:11,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 23:25:11,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 23:25:11,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 23:25:11,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 23:25:11,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 23:25:11,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 23:25:11,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 23:25:11,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 23:25:11,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 23:25:11,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 23:25:11,591 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 23:25:11,593 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 23:25:11,593 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-26 23:25:11,593 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 23:25:11,594 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-26T23:25:11,865 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be 2024-11-26 23:25:11,868 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 23:25:11,868 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T23:25:11,877 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-26T23:25:11,902 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T23:25:11,950 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T23:25:11,951 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T23:25:11,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T23:25:11,977 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351, deleteOnExit=true 2024-11-26T23:25:11,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T23:25:11,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/test.cache.data in system properties and HBase conf 2024-11-26T23:25:11,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T23:25:11,979 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/hadoop.log.dir in system properties and HBase conf 2024-11-26T23:25:11,979 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T23:25:11,980 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T23:25:11,980 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T23:25:12,058 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T23:25:12,150 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T23:25:12,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:25:12,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:25:12,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T23:25:12,156 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:25:12,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T23:25:12,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T23:25:12,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:25:12,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:25:12,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T23:25:12,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/nfs.dump.dir in system properties and HBase conf 2024-11-26T23:25:12,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/java.io.tmpdir in system properties and HBase conf 2024-11-26T23:25:12,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:25:12,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T23:25:12,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T23:25:12,965 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T23:25:13,039 INFO [Time-limited test {}] log.Log(170): Logging initialized @2341ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T23:25:13,112 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:25:13,179 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:25:13,208 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:25:13,208 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:25:13,209 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:25:13,222 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:25:13,224 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:25:13,225 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:25:13,413 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/java.io.tmpdir/jetty-localhost-33263-hadoop-hdfs-3_4_1-tests_jar-_-any-12607387964385225519/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:25:13,419 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:33263} 2024-11-26T23:25:13,420 INFO [Time-limited test {}] server.Server(415): Started @2722ms 2024-11-26T23:25:13,798 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:25:13,805 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:25:13,806 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:25:13,807 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:25:13,807 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:25:13,808 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:25:13,809 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:25:13,930 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/java.io.tmpdir/jetty-localhost-35783-hadoop-hdfs-3_4_1-tests_jar-_-any-17034664483515512852/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:25:13,931 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:35783} 2024-11-26T23:25:13,931 INFO [Time-limited test {}] server.Server(415): Started @3233ms 2024-11-26T23:25:13,985 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:25:14,114 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:25:14,123 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:25:14,125 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:25:14,125 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:25:14,125 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:25:14,127 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:25:14,127 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:25:14,285 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/java.io.tmpdir/jetty-localhost-42903-hadoop-hdfs-3_4_1-tests_jar-_-any-11917000842462406363/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:25:14,286 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:42903} 2024-11-26T23:25:14,286 INFO [Time-limited test {}] server.Server(415): Started @3588ms 2024-11-26T23:25:14,288 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:25:14,333 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:25:14,339 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:25:14,341 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:25:14,341 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:25:14,341 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:25:14,342 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:25:14,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:25:14,444 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data4/current/BP-1198857476-172.17.0.2-1732663512723/current, will proceed with Du for space computation calculation, 2024-11-26T23:25:14,444 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data2/current/BP-1198857476-172.17.0.2-1732663512723/current, will proceed with Du for space computation calculation, 2024-11-26T23:25:14,444 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data3/current/BP-1198857476-172.17.0.2-1732663512723/current, will proceed with Du for space computation calculation, 2024-11-26T23:25:14,444 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data1/current/BP-1198857476-172.17.0.2-1732663512723/current, will proceed with Du for space computation calculation, 2024-11-26T23:25:14,496 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/java.io.tmpdir/jetty-localhost-35227-hadoop-hdfs-3_4_1-tests_jar-_-any-1648775387055629732/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:25:14,497 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:35227} 2024-11-26T23:25:14,497 INFO [Time-limited test {}] server.Server(415): Started @3799ms 2024-11-26T23:25:14,499 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:25:14,500 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:25:14,500 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:25:14,576 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6b2e193b772566fb with lease ID 0x325154c4371c4a5c: Processing first storage report for DS-13793ef1-03f2-4b22-a065-26d2a150e69d from datanode DatanodeRegistration(127.0.0.1:37013, datanodeUuid=74493783-2291-407c-a260-5e7b97ce8694, infoPort=42789, infoSecurePort=0, ipcPort=46047, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723) 2024-11-26T23:25:14,578 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6b2e193b772566fb with lease ID 0x325154c4371c4a5c: from storage DS-13793ef1-03f2-4b22-a065-26d2a150e69d node DatanodeRegistration(127.0.0.1:37013, datanodeUuid=74493783-2291-407c-a260-5e7b97ce8694, infoPort=42789, infoSecurePort=0, ipcPort=46047, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T23:25:14,578 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xec27aff37e5f29f4 with lease ID 0x325154c4371c4a5b: Processing first storage report for DS-99717455-0f20-44e5-9444-1c9bdbc35fbb from datanode DatanodeRegistration(127.0.0.1:43863, datanodeUuid=b612efb3-28bb-4f5f-b069-259610adb432, infoPort=45349, infoSecurePort=0, ipcPort=36207, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723) 2024-11-26T23:25:14,578 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xec27aff37e5f29f4 with lease ID 0x325154c4371c4a5b: from storage DS-99717455-0f20-44e5-9444-1c9bdbc35fbb node DatanodeRegistration(127.0.0.1:43863, datanodeUuid=b612efb3-28bb-4f5f-b069-259610adb432, infoPort=45349, infoSecurePort=0, ipcPort=36207, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:25:14,579 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6b2e193b772566fb with lease ID 0x325154c4371c4a5c: Processing first storage report for DS-d3bd527f-6c00-48b2-b614-979c8fa9c1de from datanode DatanodeRegistration(127.0.0.1:37013, datanodeUuid=74493783-2291-407c-a260-5e7b97ce8694, infoPort=42789, infoSecurePort=0, ipcPort=46047, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723) 2024-11-26T23:25:14,579 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6b2e193b772566fb with lease ID 0x325154c4371c4a5c: from storage DS-d3bd527f-6c00-48b2-b614-979c8fa9c1de node DatanodeRegistration(127.0.0.1:37013, datanodeUuid=74493783-2291-407c-a260-5e7b97ce8694, infoPort=42789, infoSecurePort=0, ipcPort=46047, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:25:14,579 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xec27aff37e5f29f4 with lease ID 0x325154c4371c4a5b: Processing first storage report for DS-46543c97-3f93-4135-9085-69c9e0c373cc from datanode DatanodeRegistration(127.0.0.1:43863, datanodeUuid=b612efb3-28bb-4f5f-b069-259610adb432, infoPort=45349, infoSecurePort=0, ipcPort=36207, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723) 2024-11-26T23:25:14,579 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xec27aff37e5f29f4 with lease ID 0x325154c4371c4a5b: from storage DS-46543c97-3f93-4135-9085-69c9e0c373cc node DatanodeRegistration(127.0.0.1:43863, datanodeUuid=b612efb3-28bb-4f5f-b069-259610adb432, infoPort=45349, infoSecurePort=0, ipcPort=36207, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:25:14,635 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data5/current/BP-1198857476-172.17.0.2-1732663512723/current, will proceed with Du for space computation calculation, 2024-11-26T23:25:14,635 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data6/current/BP-1198857476-172.17.0.2-1732663512723/current, will proceed with Du for space computation calculation, 2024-11-26T23:25:14,663 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:25:14,668 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x40d134789700a0df with lease ID 0x325154c4371c4a5d: Processing first storage report for DS-f976893c-cf48-4bea-904b-c9f351c04507 from datanode DatanodeRegistration(127.0.0.1:36499, datanodeUuid=67a5b5e6-7d3a-49aa-9759-234547f3d9d4, infoPort=41819, infoSecurePort=0, ipcPort=43363, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723) 2024-11-26T23:25:14,668 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x40d134789700a0df with lease ID 0x325154c4371c4a5d: from storage DS-f976893c-cf48-4bea-904b-c9f351c04507 node DatanodeRegistration(127.0.0.1:36499, datanodeUuid=67a5b5e6-7d3a-49aa-9759-234547f3d9d4, infoPort=41819, infoSecurePort=0, ipcPort=43363, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:25:14,669 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x40d134789700a0df with lease ID 0x325154c4371c4a5d: Processing first storage report for DS-9def6e5c-7bd2-4de4-83c2-5c8a9cc3d7a5 from datanode DatanodeRegistration(127.0.0.1:36499, datanodeUuid=67a5b5e6-7d3a-49aa-9759-234547f3d9d4, infoPort=41819, infoSecurePort=0, ipcPort=43363, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723) 2024-11-26T23:25:14,669 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x40d134789700a0df with lease ID 0x325154c4371c4a5d: from storage DS-9def6e5c-7bd2-4de4-83c2-5c8a9cc3d7a5 node DatanodeRegistration(127.0.0.1:36499, datanodeUuid=67a5b5e6-7d3a-49aa-9759-234547f3d9d4, infoPort=41819, infoSecurePort=0, ipcPort=43363, storageInfo=lv=-57;cid=testClusterID;nsid=1042408270;c=1732663512723), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:25:14,912 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be 2024-11-26T23:25:14,991 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/zookeeper_0, clientPort=51163, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T23:25:15,003 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51163 2024-11-26T23:25:15,017 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:15,020 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:15,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:25:15,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:25:15,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:25:15,652 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e with version=8 2024-11-26T23:25:15,652 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/hbase-staging 2024-11-26T23:25:15,951 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:25:15,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:15,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:15,967 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:25:15,967 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:15,967 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:25:16,103 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T23:25:16,171 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T23:25:16,180 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T23:25:16,184 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:25:16,212 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17393 (auto-detected) 2024-11-26T23:25:16,213 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T23:25:16,233 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35437 2024-11-26T23:25:16,254 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35437 connecting to ZooKeeper ensemble=127.0.0.1:51163 2024-11-26T23:25:16,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:354370x0, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:25:16,289 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35437-0x101569901eb0000 connected 2024-11-26T23:25:16,316 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:16,319 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:16,329 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:25:16,333 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e, hbase.cluster.distributed=false 2024-11-26T23:25:16,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:25:16,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35437 2024-11-26T23:25:16,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35437 2024-11-26T23:25:16,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35437 2024-11-26T23:25:16,362 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35437 2024-11-26T23:25:16,362 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35437 2024-11-26T23:25:16,468 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:25:16,470 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:16,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:16,471 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:25:16,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:16,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:25:16,474 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:25:16,476 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:25:16,477 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42261 2024-11-26T23:25:16,479 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42261 connecting to ZooKeeper ensemble=127.0.0.1:51163 2024-11-26T23:25:16,480 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:16,482 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:16,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422610x0, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:25:16,490 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42261-0x101569901eb0001 connected 2024-11-26T23:25:16,490 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:25:16,494 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:25:16,501 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:25:16,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:25:16,508 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:25:16,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42261 2024-11-26T23:25:16,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42261 2024-11-26T23:25:16,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42261 2024-11-26T23:25:16,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42261 2024-11-26T23:25:16,511 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42261 2024-11-26T23:25:16,529 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:25:16,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:16,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:16,530 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:25:16,530 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:16,531 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:25:16,531 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:25:16,531 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:25:16,532 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34663 2024-11-26T23:25:16,534 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34663 connecting to ZooKeeper ensemble=127.0.0.1:51163 2024-11-26T23:25:16,535 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:16,538 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:16,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:346630x0, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:25:16,544 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34663-0x101569901eb0002 connected 2024-11-26T23:25:16,545 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:25:16,545 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:25:16,547 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:25:16,548 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:25:16,550 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:25:16,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34663 2024-11-26T23:25:16,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34663 2024-11-26T23:25:16,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34663 2024-11-26T23:25:16,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34663 2024-11-26T23:25:16,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34663 2024-11-26T23:25:16,570 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:25:16,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:16,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:16,571 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:25:16,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:25:16,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:25:16,571 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:25:16,571 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:25:16,572 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46791 2024-11-26T23:25:16,574 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46791 connecting to ZooKeeper ensemble=127.0.0.1:51163 2024-11-26T23:25:16,575 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:16,577 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:16,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467910x0, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:25:16,583 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46791-0x101569901eb0003 connected 2024-11-26T23:25:16,583 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:25:16,583 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:25:16,584 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:25:16,585 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:25:16,587 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:25:16,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46791 2024-11-26T23:25:16,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46791 2024-11-26T23:25:16,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46791 2024-11-26T23:25:16,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46791 2024-11-26T23:25:16,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46791 2024-11-26T23:25:16,612 DEBUG [M:0;c707e9ace675:35437 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c707e9ace675:35437 2024-11-26T23:25:16,615 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c707e9ace675,35437,1732663515757 2024-11-26T23:25:16,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:25:16,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:25:16,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:25:16,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:25:16,625 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c707e9ace675,35437,1732663515757 2024-11-26T23:25:16,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:25:16,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:16,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:25:16,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:25:16,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:16,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:16,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:16,651 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T23:25:16,652 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c707e9ace675,35437,1732663515757 from backup master directory 2024-11-26T23:25:16,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c707e9ace675,35437,1732663515757 2024-11-26T23:25:16,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:25:16,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:25:16,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:25:16,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:25:16,656 WARN [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:25:16,656 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c707e9ace675,35437,1732663515757 2024-11-26T23:25:16,659 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T23:25:16,660 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T23:25:16,716 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/hbase.id] with ID: 92411619-9446-40ba-872e-d0d817f07ae3 2024-11-26T23:25:16,717 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/.tmp/hbase.id 2024-11-26T23:25:16,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:25:16,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:25:16,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:25:16,732 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/.tmp/hbase.id]:[hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/hbase.id] 2024-11-26T23:25:16,775 INFO [master/c707e9ace675:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:25:16,780 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T23:25:16,799 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-26T23:25:16,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:16,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:16,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:16,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:16,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:25:16,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:25:16,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:25:16,841 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:25:16,843 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T23:25:16,849 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:16,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:25:16,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:25:16,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:25:16,905 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store 2024-11-26T23:25:16,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:25:16,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:25:16,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:25:16,935 INFO [master/c707e9ace675:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T23:25:16,938 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:16,939 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:25:16,939 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:25:16,940 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:25:16,941 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:25:16,941 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:25:16,942 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:25:16,943 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663516939Disabling compacts and flushes for region at 1732663516939Disabling writes for close at 1732663516941 (+2 ms)Writing region close event to WAL at 1732663516942 (+1 ms)Closed at 1732663516942 2024-11-26T23:25:16,945 WARN [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/.initializing 2024-11-26T23:25:16,945 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/WALs/c707e9ace675,35437,1732663515757 2024-11-26T23:25:16,953 INFO [master/c707e9ace675:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:16,968 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C35437%2C1732663515757, suffix=, logDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/WALs/c707e9ace675,35437,1732663515757, archiveDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/oldWALs, maxLogs=10 2024-11-26T23:25:16,998 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/WALs/c707e9ace675,35437,1732663515757/c707e9ace675%2C35437%2C1732663515757.1732663516973, exclude list is [], retry=0 2024-11-26T23:25:17,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:17,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:17,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:17,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:17,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T23:25:17,068 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/WALs/c707e9ace675,35437,1732663515757/c707e9ace675%2C35437%2C1732663515757.1732663516973 2024-11-26T23:25:17,069 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:17,070 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:17,070 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:17,073 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,074 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T23:25:17,146 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:17,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:17,150 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,153 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T23:25:17,153 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:17,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:17,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T23:25:17,157 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:17,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:17,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T23:25:17,161 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:17,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:17,162 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,165 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,166 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,171 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,171 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,174 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:25:17,177 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:25:17,181 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:17,183 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67069868, jitterRate=-5.810856819152832E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:25:17,188 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732663517086Initializing all the Stores at 1732663517088 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663517089 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663517090 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663517090Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663517090Cleaning up temporary data from old regions at 1732663517171 (+81 ms)Region opened successfully at 1732663517188 (+17 ms) 2024-11-26T23:25:17,189 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T23:25:17,229 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6095cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:25:17,262 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T23:25:17,274 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T23:25:17,274 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T23:25:17,278 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T23:25:17,279 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T23:25:17,284 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T23:25:17,284 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T23:25:17,310 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T23:25:17,319 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T23:25:17,321 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T23:25:17,323 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T23:25:17,324 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T23:25:17,327 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T23:25:17,329 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T23:25:17,332 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T23:25:17,334 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T23:25:17,335 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T23:25:17,337 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T23:25:17,355 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T23:25:17,357 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T23:25:17,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:25:17,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:25:17,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:25:17,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:25:17,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,365 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c707e9ace675,35437,1732663515757, sessionid=0x101569901eb0000, setting cluster-up flag (Was=false) 2024-11-26T23:25:17,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,383 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T23:25:17,385 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,35437,1732663515757 2024-11-26T23:25:17,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:17,396 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T23:25:17,398 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,35437,1732663515757 2024-11-26T23:25:17,404 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T23:25:17,480 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T23:25:17,489 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T23:25:17,496 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T23:25:17,496 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(746): ClusterId : 92411619-9446-40ba-872e-d0d817f07ae3 2024-11-26T23:25:17,496 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(746): ClusterId : 92411619-9446-40ba-872e-d0d817f07ae3 2024-11-26T23:25:17,496 INFO [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(746): ClusterId : 92411619-9446-40ba-872e-d0d817f07ae3 2024-11-26T23:25:17,499 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:25:17,499 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:25:17,499 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:25:17,505 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:25:17,505 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:25:17,505 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:25:17,505 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:25:17,505 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:25:17,505 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:25:17,502 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c707e9ace675,35437,1732663515757 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T23:25:17,510 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:25:17,510 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:25:17,510 DEBUG [RS:1;c707e9ace675:34663 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443834cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:25:17,510 DEBUG [RS:2;c707e9ace675:46791 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d3610b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:25:17,511 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:25:17,511 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:25:17,511 DEBUG [RS:0;c707e9ace675:42261 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547e976a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:25:17,511 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:25:17,512 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:25:17,512 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:25:17,512 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c707e9ace675:0, corePoolSize=10, maxPoolSize=10 2024-11-26T23:25:17,512 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,512 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:25:17,512 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,521 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:25:17,522 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T23:25:17,528 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:17,528 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T23:25:17,529 DEBUG [RS:0;c707e9ace675:42261 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c707e9ace675:42261 2024-11-26T23:25:17,530 DEBUG [RS:2;c707e9ace675:46791 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;c707e9ace675:46791 2024-11-26T23:25:17,531 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732663547530 2024-11-26T23:25:17,532 DEBUG [RS:1;c707e9ace675:34663 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c707e9ace675:34663 2024-11-26T23:25:17,532 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T23:25:17,533 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T23:25:17,533 INFO [RS:2;c707e9ace675:46791 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:25:17,533 INFO [RS:1;c707e9ace675:34663 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:25:17,533 INFO [RS:0;c707e9ace675:42261 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:25:17,534 INFO [RS:2;c707e9ace675:46791 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:25:17,534 INFO [RS:1;c707e9ace675:34663 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:25:17,534 INFO [RS:0;c707e9ace675:42261 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:25:17,534 DEBUG [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:25:17,534 DEBUG [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:25:17,534 DEBUG [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:25:17,537 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,35437,1732663515757 with port=42261, startcode=1732663516429 2024-11-26T23:25:17,537 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T23:25:17,537 INFO [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,35437,1732663515757 with port=34663, startcode=1732663516528 2024-11-26T23:25:17,537 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,35437,1732663515757 with port=46791, startcode=1732663516570 2024-11-26T23:25:17,537 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T23:25:17,538 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T23:25:17,538 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T23:25:17,539 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,544 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T23:25:17,545 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T23:25:17,545 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T23:25:17,549 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T23:25:17,550 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T23:25:17,551 DEBUG [RS:1;c707e9ace675:34663 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:25:17,551 DEBUG [RS:2;c707e9ace675:46791 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:25:17,551 DEBUG [RS:0;c707e9ace675:42261 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:25:17,552 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663517551,5,FailOnTimeoutGroup] 2024-11-26T23:25:17,553 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663517553,5,FailOnTimeoutGroup] 2024-11-26T23:25:17,553 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,553 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T23:25:17,554 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,555 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:25:17,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:25:17,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:25:17,559 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T23:25:17,560 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e 2024-11-26T23:25:17,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:25:17,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:25:17,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:25:17,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:17,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:25:17,613 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:25:17,613 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:17,614 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53225, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:25:17,614 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43679, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:25:17,614 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42341, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:25:17,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:17,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:25:17,617 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:25:17,617 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:17,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:17,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:25:17,621 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:25:17,621 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:17,622 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35437 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,46791,1732663516570 2024-11-26T23:25:17,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:17,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:25:17,625 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35437 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,46791,1732663516570 2024-11-26T23:25:17,625 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:25:17,625 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:17,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:17,626 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:25:17,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740 2024-11-26T23:25:17,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740 2024-11-26T23:25:17,631 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:25:17,631 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:25:17,633 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:25:17,635 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:25:17,638 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35437 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,34663,1732663516528 2024-11-26T23:25:17,638 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35437 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,34663,1732663516528 2024-11-26T23:25:17,640 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:17,642 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61060630, jitterRate=-0.09012570977210999}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:25:17,642 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35437 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,42261,1732663516429 2024-11-26T23:25:17,642 DEBUG [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e 2024-11-26T23:25:17,642 DEBUG [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45925 2024-11-26T23:25:17,642 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35437 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,42261,1732663516429 2024-11-26T23:25:17,642 DEBUG [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:25:17,642 DEBUG [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e 2024-11-26T23:25:17,643 DEBUG [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45925 2024-11-26T23:25:17,643 DEBUG [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:25:17,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732663517607Initializing all the Stores at 1732663517609 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663517609Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663517610 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663517610Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663517610Cleaning up temporary data from old regions at 1732663517631 (+21 ms)Region opened successfully at 1732663517645 (+14 ms) 2024-11-26T23:25:17,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:25:17,646 DEBUG [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e 2024-11-26T23:25:17,646 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:25:17,646 DEBUG [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45925 2024-11-26T23:25:17,646 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:25:17,646 DEBUG [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:25:17,646 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:25:17,646 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:25:17,647 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:25:17,648 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663517645Disabling compacts and flushes for region at 1732663517645Disabling writes for close at 1732663517646 (+1 ms)Writing region close event to WAL at 1732663517647 (+1 ms)Closed at 1732663517647 2024-11-26T23:25:17,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:25:17,650 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:25:17,651 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T23:25:17,655 DEBUG [RS:1;c707e9ace675:34663 {}] zookeeper.ZKUtil(111): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,34663,1732663516528 2024-11-26T23:25:17,655 DEBUG [RS:2;c707e9ace675:46791 {}] zookeeper.ZKUtil(111): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,46791,1732663516570 2024-11-26T23:25:17,655 WARN [RS:1;c707e9ace675:34663 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:25:17,655 WARN [RS:2;c707e9ace675:46791 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:25:17,655 INFO [RS:1;c707e9ace675:34663 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:17,655 INFO [RS:2;c707e9ace675:46791 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:17,655 DEBUG [RS:0;c707e9ace675:42261 {}] zookeeper.ZKUtil(111): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,42261,1732663516429 2024-11-26T23:25:17,655 WARN [RS:0;c707e9ace675:42261 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:25:17,656 DEBUG [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,34663,1732663516528 2024-11-26T23:25:17,656 DEBUG [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,46791,1732663516570 2024-11-26T23:25:17,656 INFO [RS:0;c707e9ace675:42261 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:17,656 DEBUG [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429 2024-11-26T23:25:17,657 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,42261,1732663516429] 2024-11-26T23:25:17,657 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,46791,1732663516570] 2024-11-26T23:25:17,657 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,34663,1732663516528] 2024-11-26T23:25:17,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T23:25:17,670 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:25:17,674 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T23:25:17,685 INFO [RS:2;c707e9ace675:46791 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:25:17,685 INFO [RS:0;c707e9ace675:42261 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:25:17,685 INFO [RS:1;c707e9ace675:34663 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:25:17,698 INFO [RS:0;c707e9ace675:42261 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:25:17,698 INFO [RS:2;c707e9ace675:46791 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:25:17,698 INFO [RS:1;c707e9ace675:34663 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:25:17,704 INFO [RS:0;c707e9ace675:42261 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:25:17,704 INFO [RS:2;c707e9ace675:46791 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:25:17,704 INFO [RS:1;c707e9ace675:34663 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:25:17,704 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,704 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,704 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,705 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:25:17,706 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:25:17,706 INFO [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:25:17,711 INFO [RS:2;c707e9ace675:46791 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:25:17,711 INFO [RS:0;c707e9ace675:42261 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:25:17,711 INFO [RS:1;c707e9ace675:34663 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:25:17,713 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,713 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,713 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,713 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,713 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,713 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,713 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,713 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,713 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,713 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:25:17,714 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:25:17,714 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:25:17,714 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,714 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:25:17,715 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:2;c707e9ace675:46791 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:25:17,715 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:25:17,715 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:25:17,715 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:25:17,716 DEBUG [RS:0;c707e9ace675:42261 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:25:17,716 DEBUG [RS:1;c707e9ace675:34663 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:25:17,719 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,719 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,719 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,719 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,719 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,719 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34663,1732663516528-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:25:17,720 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,720 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,721 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,721 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,721 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,721 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,721 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,721 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,721 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,46791,1732663516570-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:25:17,721 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,721 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,721 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42261,1732663516429-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:25:17,740 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:25:17,741 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,46791,1732663516570-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,742 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,742 INFO [RS:2;c707e9ace675:46791 {}] regionserver.Replication(171): c707e9ace675,46791,1732663516570 started 2024-11-26T23:25:17,746 INFO [RS:1;c707e9ace675:34663 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:25:17,746 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34663,1732663516528-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,746 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:25:17,746 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,746 INFO [RS:1;c707e9ace675:34663 {}] regionserver.Replication(171): c707e9ace675,34663,1732663516528 started 2024-11-26T23:25:17,746 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42261,1732663516429-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,747 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,747 INFO [RS:0;c707e9ace675:42261 {}] regionserver.Replication(171): c707e9ace675,42261,1732663516429 started 2024-11-26T23:25:17,762 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,763 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,46791,1732663516570, RpcServer on c707e9ace675/172.17.0.2:46791, sessionid=0x101569901eb0003 2024-11-26T23:25:17,764 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:25:17,764 DEBUG [RS:2;c707e9ace675:46791 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,46791,1732663516570 2024-11-26T23:25:17,764 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,46791,1732663516570' 2024-11-26T23:25:17,764 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:25:17,766 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:25:17,766 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:25:17,766 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:25:17,767 DEBUG [RS:2;c707e9ace675:46791 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,46791,1732663516570 2024-11-26T23:25:17,767 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,46791,1732663516570' 2024-11-26T23:25:17,767 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:25:17,767 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:25:17,768 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,768 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,42261,1732663516429, RpcServer on c707e9ace675/172.17.0.2:42261, sessionid=0x101569901eb0001 2024-11-26T23:25:17,768 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:25:17,768 DEBUG [RS:2;c707e9ace675:46791 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:25:17,768 DEBUG [RS:0;c707e9ace675:42261 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,42261,1732663516429 2024-11-26T23:25:17,769 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,42261,1732663516429' 2024-11-26T23:25:17,769 INFO [RS:2;c707e9ace675:46791 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:25:17,769 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:25:17,769 INFO [RS:2;c707e9ace675:46791 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:25:17,769 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:25:17,770 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:17,770 INFO [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,34663,1732663516528, RpcServer on c707e9ace675/172.17.0.2:34663, sessionid=0x101569901eb0002 2024-11-26T23:25:17,770 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:25:17,770 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:25:17,770 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:25:17,770 DEBUG [RS:0;c707e9ace675:42261 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,42261,1732663516429 2024-11-26T23:25:17,770 DEBUG [RS:1;c707e9ace675:34663 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,34663,1732663516528 2024-11-26T23:25:17,770 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,42261,1732663516429' 2024-11-26T23:25:17,770 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,34663,1732663516528' 2024-11-26T23:25:17,770 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:25:17,770 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:25:17,771 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:25:17,771 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:25:17,772 DEBUG [RS:0;c707e9ace675:42261 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:25:17,772 INFO [RS:0;c707e9ace675:42261 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:25:17,772 INFO [RS:0;c707e9ace675:42261 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:25:17,772 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:25:17,772 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:25:17,772 DEBUG [RS:1;c707e9ace675:34663 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,34663,1732663516528 2024-11-26T23:25:17,772 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,34663,1732663516528' 2024-11-26T23:25:17,772 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:25:17,773 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:25:17,774 DEBUG [RS:1;c707e9ace675:34663 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:25:17,774 INFO [RS:1;c707e9ace675:34663 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:25:17,774 INFO [RS:1;c707e9ace675:34663 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:25:17,826 WARN [c707e9ace675:35437 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T23:25:17,876 INFO [RS:0;c707e9ace675:42261 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:17,876 INFO [RS:2;c707e9ace675:46791 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:17,876 INFO [RS:1;c707e9ace675:34663 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:17,881 INFO [RS:0;c707e9ace675:42261 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C42261%2C1732663516429, suffix=, logDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429, archiveDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/oldWALs, maxLogs=32 2024-11-26T23:25:17,881 INFO [RS:1;c707e9ace675:34663 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C34663%2C1732663516528, suffix=, logDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,34663,1732663516528, archiveDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/oldWALs, maxLogs=32 2024-11-26T23:25:17,881 INFO [RS:2;c707e9ace675:46791 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C46791%2C1732663516570, suffix=, logDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,46791,1732663516570, archiveDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/oldWALs, maxLogs=32 2024-11-26T23:25:17,908 DEBUG [RS:0;c707e9ace675:42261 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429/c707e9ace675%2C42261%2C1732663516429.1732663517892, exclude list is [], retry=0 2024-11-26T23:25:17,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:17,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:17,914 DEBUG [RS:2;c707e9ace675:46791 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,46791,1732663516570/c707e9ace675%2C46791%2C1732663516570.1732663517893, exclude list is [], retry=0 2024-11-26T23:25:17,914 DEBUG [RS:1;c707e9ace675:34663 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,34663,1732663516528/c707e9ace675%2C34663%2C1732663516528.1732663517893, exclude list is [], retry=0 2024-11-26T23:25:17,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:17,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:17,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:17,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:17,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:17,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:17,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:17,926 INFO [RS:0;c707e9ace675:42261 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429/c707e9ace675%2C42261%2C1732663516429.1732663517892 2024-11-26T23:25:17,930 DEBUG [RS:0;c707e9ace675:42261 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:17,941 INFO [RS:2;c707e9ace675:46791 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,46791,1732663516570/c707e9ace675%2C46791%2C1732663516570.1732663517893 2024-11-26T23:25:17,941 INFO [RS:1;c707e9ace675:34663 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,34663,1732663516528/c707e9ace675%2C34663%2C1732663516528.1732663517893 2024-11-26T23:25:17,942 DEBUG [RS:2;c707e9ace675:46791 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:17,944 DEBUG [RS:1;c707e9ace675:34663 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:18,078 DEBUG [c707e9ace675:35437 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-26T23:25:18,087 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(204): Hosts are {c707e9ace675=0} racks are {/default-rack=0} 2024-11-26T23:25:18,093 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T23:25:18,093 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T23:25:18,093 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T23:25:18,093 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T23:25:18,093 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T23:25:18,093 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T23:25:18,093 INFO [c707e9ace675:35437 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T23:25:18,093 INFO [c707e9ace675:35437 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T23:25:18,093 INFO [c707e9ace675:35437 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T23:25:18,093 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T23:25:18,101 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c707e9ace675,42261,1732663516429 2024-11-26T23:25:18,107 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,42261,1732663516429, state=OPENING 2024-11-26T23:25:18,112 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T23:25:18,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:18,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:18,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:18,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:18,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:18,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:18,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:18,115 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:18,117 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:25:18,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,42261,1732663516429}] 2024-11-26T23:25:18,293 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:25:18,296 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41305, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:25:18,308 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T23:25:18,309 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:18,310 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T23:25:18,313 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C42261%2C1732663516429.meta, suffix=.meta, logDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429, archiveDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/oldWALs, maxLogs=32 2024-11-26T23:25:18,329 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta, exclude list is [], retry=0 2024-11-26T23:25:18,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:18,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:18,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:18,336 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta 2024-11-26T23:25:18,336 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:18,337 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:18,338 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T23:25:18,341 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T23:25:18,346 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T23:25:18,351 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T23:25:18,351 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:18,351 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T23:25:18,351 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T23:25:18,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:25:18,356 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:25:18,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:18,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:18,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:25:18,358 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:25:18,359 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:18,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:18,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:25:18,361 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:25:18,361 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:18,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:18,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:25:18,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:25:18,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:18,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:18,365 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:25:18,366 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740 2024-11-26T23:25:18,368 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740 2024-11-26T23:25:18,371 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:25:18,371 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:25:18,372 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:25:18,374 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:25:18,376 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67218724, jitterRate=0.0016370415687561035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:25:18,376 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T23:25:18,378 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732663518352Writing region info on filesystem at 1732663518352Initializing all the Stores at 1732663518354 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663518354Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663518354Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663518354Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663518354Cleaning up temporary data from old regions at 1732663518371 (+17 ms)Running coprocessor post-open hooks at 1732663518376 (+5 ms)Region opened successfully at 1732663518378 (+2 ms) 2024-11-26T23:25:18,385 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732663518284 2024-11-26T23:25:18,396 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T23:25:18,397 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T23:25:18,398 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,42261,1732663516429 2024-11-26T23:25:18,400 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,42261,1732663516429, state=OPEN 2024-11-26T23:25:18,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:25:18,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:25:18,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:25:18,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:25:18,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:18,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:18,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:18,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:18,403 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c707e9ace675,42261,1732663516429 2024-11-26T23:25:18,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T23:25:18,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,42261,1732663516429 in 285 msec 2024-11-26T23:25:18,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T23:25:18,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 753 msec 2024-11-26T23:25:18,417 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:25:18,417 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T23:25:18,439 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:25:18,440 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1] 2024-11-26T23:25:18,461 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:25:18,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53973, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:25:18,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0540 sec 2024-11-26T23:25:18,486 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732663518486, completionTime=-1 2024-11-26T23:25:18,489 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-26T23:25:18,489 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T23:25:18,514 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-26T23:25:18,514 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732663578514 2024-11-26T23:25:18,514 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732663638514 2024-11-26T23:25:18,514 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-26T23:25:18,515 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T23:25:18,523 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35437,1732663515757-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:18,523 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35437,1732663515757-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:18,523 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35437,1732663515757-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:18,525 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c707e9ace675:35437, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:18,526 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:18,526 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:18,533 DEBUG [master/c707e9ace675:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T23:25:18,556 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.899sec 2024-11-26T23:25:18,558 INFO [master/c707e9ace675:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T23:25:18,559 INFO [master/c707e9ace675:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T23:25:18,560 INFO [master/c707e9ace675:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T23:25:18,561 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T23:25:18,561 INFO [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T23:25:18,562 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35437,1732663515757-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:25:18,562 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35437,1732663515757-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T23:25:18,567 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T23:25:18,568 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T23:25:18,568 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,35437,1732663515757-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:18,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21867609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:25:18,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,35437,-1 for getting cluster id 2024-11-26T23:25:18,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:25:18,648 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '92411619-9446-40ba-872e-d0d817f07ae3' 2024-11-26T23:25:18,651 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:25:18,651 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "92411619-9446-40ba-872e-d0d817f07ae3" 2024-11-26T23:25:18,652 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28dd8c33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:25:18,652 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,35437,-1] 2024-11-26T23:25:18,655 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:25:18,657 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:25:18,658 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:25:18,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603109e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:25:18,662 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:25:18,670 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1] 2024-11-26T23:25:18,670 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:25:18,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:25:18,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c707e9ace675,35437,1732663515757 2024-11-26T23:25:18,694 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:45925/hbase 2024-11-26T23:25:18,710 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=363, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=7304 2024-11-26T23:25:18,731 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:18,734 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:18,735 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:18,739 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-76369184, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/hregion-76369184, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:18,761 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-76369184/hregion-76369184.1732663518741, exclude list is [], retry=0 2024-11-26T23:25:18,765 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:18,766 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:18,766 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:18,769 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-76369184/hregion-76369184.1732663518741 2024-11-26T23:25:18,770 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:18,770 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 03702a67ffe2098cd437a04654625d8a, NAME => 'testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/hbase 2024-11-26T23:25:18,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741838_1014 (size=64) 2024-11-26T23:25:18,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741838_1014 (size=64) 2024-11-26T23:25:18,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741838_1014 (size=64) 2024-11-26T23:25:18,785 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:18,787 INFO [StoreOpener-03702a67ffe2098cd437a04654625d8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,789 INFO [StoreOpener-03702a67ffe2098cd437a04654625d8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03702a67ffe2098cd437a04654625d8a columnFamilyName a 2024-11-26T23:25:18,789 DEBUG [StoreOpener-03702a67ffe2098cd437a04654625d8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:18,790 INFO [StoreOpener-03702a67ffe2098cd437a04654625d8a-1 {}] regionserver.HStore(327): Store=03702a67ffe2098cd437a04654625d8a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:18,790 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,791 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,792 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,792 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,793 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,795 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,798 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:18,799 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 03702a67ffe2098cd437a04654625d8a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67277810, jitterRate=0.002517491579055786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:25:18,801 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 03702a67ffe2098cd437a04654625d8a: Writing region info on filesystem at 1732663518785Initializing all the Stores at 1732663518787 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663518787Cleaning up temporary data from old regions at 1732663518793 (+6 ms)Region opened successfully at 1732663518801 (+8 ms) 2024-11-26T23:25:18,801 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 03702a67ffe2098cd437a04654625d8a, disabling compactions & flushes 2024-11-26T23:25:18,801 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a. 2024-11-26T23:25:18,801 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a. 2024-11-26T23:25:18,801 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a. after waiting 0 ms 2024-11-26T23:25:18,801 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a. 2024-11-26T23:25:18,802 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a. 2024-11-26T23:25:18,802 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 03702a67ffe2098cd437a04654625d8a: Waiting for close lock at 1732663518801Disabling compacts and flushes for region at 1732663518801Disabling writes for close at 1732663518801Writing region close event to WAL at 1732663518802 (+1 ms)Closed at 1732663518802 2024-11-26T23:25:18,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741837_1013 (size=93) 2024-11-26T23:25:18,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741837_1013 (size=93) 2024-11-26T23:25:18,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741837_1013 (size=93) 2024-11-26T23:25:18,813 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:18,813 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-76369184:(num 1732663518741) 2024-11-26T23:25:18,815 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T23:25:18,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741839_1015 (size=369) 2024-11-26T23:25:18,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741839_1015 (size=369) 2024-11-26T23:25:18,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741839_1015 (size=369) 2024-11-26T23:25:18,832 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T23:25:18,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741840_1016 (size=231) 2024-11-26T23:25:18,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741840_1016 (size=231) 2024-11-26T23:25:18,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741840_1016 (size=231) 2024-11-26T23:25:18,864 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-1, size=369 (369bytes) 2024-11-26T23:25:18,865 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-26T23:25:18,865 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-26T23:25:18,866 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-1 2024-11-26T23:25:18,871 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-1 after 3ms 2024-11-26T23:25:18,877 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-1 took 14ms 2024-11-26T23:25:18,880 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-1 so closing down 2024-11-26T23:25:18,884 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-26T23:25:18,885 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:18,886 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000001-wal-1.temp 2024-11-26T23:25:18,887 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:18,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741841_1017 (size=369) 2024-11-26T23:25:18,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741841_1017 (size=369) 2024-11-26T23:25:18,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741841_1017 (size=369) 2024-11-26T23:25:18,904 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:18,907 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000002 2024-11-26T23:25:18,910 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-26T23:25:18,910 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-1, journal: Splitting hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-1, size=369 (369bytes) at 1732663518865Finishing writing output for hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-1 so closing down at 1732663518880 (+15 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000001-wal-1.temp at 1732663518886 (+6 ms)3 split writer threads finished at 1732663518887 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732663518904 (+17 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000002 at 1732663518907 (+3 ms)Processed 2 edits across 1 Regions in 29 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732663518910 (+3 ms) 2024-11-26T23:25:18,926 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-2, size=231 (231bytes) 2024-11-26T23:25:18,926 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-2 2024-11-26T23:25:18,927 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-2 after 1ms 2024-11-26T23:25:18,931 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-2 took 5ms 2024-11-26T23:25:18,933 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-2 so closing down 2024-11-26T23:25:18,933 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:18,936 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-26T23:25:18,937 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000002-wal-2.temp 2024-11-26T23:25:18,937 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:18,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741842_1018 (size=231) 2024-11-26T23:25:18,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741842_1018 (size=231) 2024-11-26T23:25:18,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741842_1018 (size=231) 2024-11-26T23:25:18,947 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:18,953 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-26T23:25:18,955 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-26T23:25:18,955 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-2, journal: Splitting hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-2, size=231 (231bytes) at 1732663518926Finishing writing output for hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-2 so closing down at 1732663518933 (+7 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000002-wal-2.temp at 1732663518937 (+4 ms)3 split writer threads finished at 1732663518938 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732663518947 (+9 ms)Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732663518955 (+8 ms) 2024-11-26T23:25:18,955 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:18,957 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:18,971 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal.1732663518958, exclude list is [], retry=0 2024-11-26T23:25:18,975 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:18,976 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:18,976 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:18,978 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal.1732663518958 2024-11-26T23:25:18,980 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:18,980 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 03702a67ffe2098cd437a04654625d8a, NAME => 'testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:18,980 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:18,981 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,981 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,983 INFO [StoreOpener-03702a67ffe2098cd437a04654625d8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,984 INFO [StoreOpener-03702a67ffe2098cd437a04654625d8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03702a67ffe2098cd437a04654625d8a columnFamilyName a 2024-11-26T23:25:18,984 DEBUG [StoreOpener-03702a67ffe2098cd437a04654625d8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:18,985 INFO [StoreOpener-03702a67ffe2098cd437a04654625d8a-1 {}] regionserver.HStore(327): Store=03702a67ffe2098cd437a04654625d8a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:18,986 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,987 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,990 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:18,991 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000002 2024-11-26T23:25:18,999 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000002 2024-11-26T23:25:19,002 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 03702a67ffe2098cd437a04654625d8a 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-26T23:25:19,053 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/.tmp/a/dd54b8804a4d4f11867f17a15dc2a164 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732663518813/Put/seqid=0 2024-11-26T23:25:19,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741844_1020 (size=5170) 2024-11-26T23:25:19,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741844_1020 (size=5170) 2024-11-26T23:25:19,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741844_1020 (size=5170) 2024-11-26T23:25:19,068 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/.tmp/a/dd54b8804a4d4f11867f17a15dc2a164 2024-11-26T23:25:19,117 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/.tmp/a/dd54b8804a4d4f11867f17a15dc2a164 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/a/dd54b8804a4d4f11867f17a15dc2a164 2024-11-26T23:25:19,126 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/a/dd54b8804a4d4f11867f17a15dc2a164, entries=2, sequenceid=2, filesize=5.0 K 2024-11-26T23:25:19,131 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 03702a67ffe2098cd437a04654625d8a in 127ms, sequenceid=2, compaction requested=false; wal=null 2024-11-26T23:25:19,132 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/0000000000000000002 2024-11-26T23:25:19,133 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:19,133 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:19,136 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 03702a67ffe2098cd437a04654625d8a 2024-11-26T23:25:19,139 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/03702a67ffe2098cd437a04654625d8a/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-26T23:25:19,140 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 03702a67ffe2098cd437a04654625d8a; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59258320, jitterRate=-0.11698222160339355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:25:19,141 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 03702a67ffe2098cd437a04654625d8a: Writing region info on filesystem at 1732663518981Initializing all the Stores at 1732663518982 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663518982Obtaining lock to block concurrent updates at 1732663519002 (+20 ms)Preparing flush snapshotting stores in 03702a67ffe2098cd437a04654625d8a at 1732663519002Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732663519005 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732663518732.03702a67ffe2098cd437a04654625d8a. at 1732663519005Flushing 03702a67ffe2098cd437a04654625d8a/a: creating writer at 1732663519006 (+1 ms)Flushing 03702a67ffe2098cd437a04654625d8a/a: appending metadata at 1732663519045 (+39 ms)Flushing 03702a67ffe2098cd437a04654625d8a/a: closing flushed file at 1732663519047 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e9aafcb: reopening flushed file at 1732663519115 (+68 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 03702a67ffe2098cd437a04654625d8a in 127ms, sequenceid=2, compaction requested=false; wal=null at 1732663519131 (+16 ms)Cleaning up temporary data from old regions at 1732663519133 (+2 ms)Region opened successfully at 1732663519141 (+8 ms) 2024-11-26T23:25:19,165 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=375 (was 363) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:43738 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:45898 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:46080 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:39744 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:39762 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:45925/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 335), ProcessCount=11 (was 11), AvailableMemoryMB=7294 (was 7304) 2024-11-26T23:25:19,176 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=375, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=7294 2024-11-26T23:25:19,191 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:19,194 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:19,194 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:19,198 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95227203, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/hregion-95227203, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:19,211 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95227203/hregion-95227203.1732663519198, exclude list is [], retry=0 2024-11-26T23:25:19,215 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:19,215 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:19,215 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:19,218 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95227203/hregion-95227203.1732663519198 2024-11-26T23:25:19,218 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:19,219 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 675c287fd51b34e2396cf85bae672631, NAME => 'testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/hbase 2024-11-26T23:25:19,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741846_1022 (size=64) 2024-11-26T23:25:19,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741846_1022 (size=64) 2024-11-26T23:25:19,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741846_1022 (size=64) 2024-11-26T23:25:19,234 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:19,236 INFO [StoreOpener-675c287fd51b34e2396cf85bae672631-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,238 INFO [StoreOpener-675c287fd51b34e2396cf85bae672631-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 675c287fd51b34e2396cf85bae672631 columnFamilyName a 2024-11-26T23:25:19,238 DEBUG [StoreOpener-675c287fd51b34e2396cf85bae672631-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:19,238 INFO [StoreOpener-675c287fd51b34e2396cf85bae672631-1 {}] regionserver.HStore(327): Store=675c287fd51b34e2396cf85bae672631/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:19,239 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,239 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,240 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,241 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,241 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,243 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,246 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:19,247 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 675c287fd51b34e2396cf85bae672631; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70583760, jitterRate=0.051779985427856445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:25:19,247 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 675c287fd51b34e2396cf85bae672631: Writing region info on filesystem at 1732663519234Initializing all the Stores at 1732663519235 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663519235Cleaning up temporary data from old regions at 1732663519241 (+6 ms)Region opened successfully at 1732663519247 (+6 ms) 2024-11-26T23:25:19,247 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 675c287fd51b34e2396cf85bae672631, disabling compactions & flushes 2024-11-26T23:25:19,247 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631. 2024-11-26T23:25:19,247 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631. 2024-11-26T23:25:19,247 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631. after waiting 0 ms 2024-11-26T23:25:19,247 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631. 2024-11-26T23:25:19,248 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631. 2024-11-26T23:25:19,248 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 675c287fd51b34e2396cf85bae672631: Waiting for close lock at 1732663519247Disabling compacts and flushes for region at 1732663519247Disabling writes for close at 1732663519247Writing region close event to WAL at 1732663519247Closed at 1732663519247 2024-11-26T23:25:19,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741845_1021 (size=93) 2024-11-26T23:25:19,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741845_1021 (size=93) 2024-11-26T23:25:19,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741845_1021 (size=93) 2024-11-26T23:25:19,254 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:19,254 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95227203:(num 1732663519198) 2024-11-26T23:25:19,255 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T23:25:19,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741847_1023 (size=369) 2024-11-26T23:25:19,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741847_1023 (size=369) 2024-11-26T23:25:19,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741847_1023 (size=369) 2024-11-26T23:25:19,269 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-26T23:25:19,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741848_1024 (size=231) 2024-11-26T23:25:19,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741848_1024 (size=231) 2024-11-26T23:25:19,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741848_1024 (size=231) 2024-11-26T23:25:19,303 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-2, size=231 (231bytes) 2024-11-26T23:25:19,303 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-2 2024-11-26T23:25:19,304 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-2 after 1ms 2024-11-26T23:25:19,308 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-2 took 5ms 2024-11-26T23:25:19,310 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-2 so closing down 2024-11-26T23:25:19,310 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:19,312 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-26T23:25:19,313 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002-wal-2.temp 2024-11-26T23:25:19,314 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:19,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741849_1025 (size=231) 2024-11-26T23:25:19,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741849_1025 (size=231) 2024-11-26T23:25:19,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741849_1025 (size=231) 2024-11-26T23:25:19,322 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:19,324 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002 2024-11-26T23:25:19,324 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-26T23:25:19,324 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-2, journal: Splitting hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-2, size=231 (231bytes) at 1732663519303Finishing writing output for hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-2 so closing down at 1732663519310 (+7 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002-wal-2.temp at 1732663519313 (+3 ms)3 split writer threads finished at 1732663519314 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732663519322 (+8 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002 at 1732663519324 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1732663519324 2024-11-26T23:25:19,339 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-1, size=369 (369bytes) 2024-11-26T23:25:19,339 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-1 2024-11-26T23:25:19,340 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-1 after 1ms 2024-11-26T23:25:19,344 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-1 took 5ms 2024-11-26T23:25:19,346 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-1 so closing down 2024-11-26T23:25:19,346 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:19,348 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-26T23:25:19,349 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000001-wal-1.temp 2024-11-26T23:25:19,349 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:19,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741850_1026 (size=369) 2024-11-26T23:25:19,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741850_1026 (size=369) 2024-11-26T23:25:19,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741850_1026 (size=369) 2024-11-26T23:25:19,359 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:19,365 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002, length=231 2024-11-26T23:25:19,367 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002 2024-11-26T23:25:19,367 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-26T23:25:19,367 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-1, journal: Splitting hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-1, size=369 (369bytes) at 1732663519339Finishing writing output for hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-1 so closing down at 1732663519346 (+7 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000001-wal-1.temp at 1732663519349 (+3 ms)3 split writer threads finished at 1732663519349Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732663519360 (+11 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002 at 1732663519367 (+7 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1732663519367 2024-11-26T23:25:19,368 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:19,370 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:19,385 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal.1732663519371, exclude list is [], retry=0 2024-11-26T23:25:19,388 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:19,389 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:19,389 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:19,391 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal.1732663519371 2024-11-26T23:25:19,392 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:19,392 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 675c287fd51b34e2396cf85bae672631, NAME => 'testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:19,392 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:19,392 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,392 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,394 INFO [StoreOpener-675c287fd51b34e2396cf85bae672631-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,396 INFO [StoreOpener-675c287fd51b34e2396cf85bae672631-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 675c287fd51b34e2396cf85bae672631 columnFamilyName a 2024-11-26T23:25:19,396 DEBUG [StoreOpener-675c287fd51b34e2396cf85bae672631-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:19,397 INFO [StoreOpener-675c287fd51b34e2396cf85bae672631-1 {}] regionserver.HStore(327): Store=675c287fd51b34e2396cf85bae672631/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:19,397 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,398 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,400 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,401 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002 2024-11-26T23:25:19,406 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002 2024-11-26T23:25:19,406 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 675c287fd51b34e2396cf85bae672631 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-26T23:25:19,423 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/.tmp/a/85d3d46c0a8b43f98dd3a30514edb425 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732663519254/Put/seqid=0 2024-11-26T23:25:19,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741852_1028 (size=5170) 2024-11-26T23:25:19,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741852_1028 (size=5170) 2024-11-26T23:25:19,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741852_1028 (size=5170) 2024-11-26T23:25:19,432 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/.tmp/a/85d3d46c0a8b43f98dd3a30514edb425 2024-11-26T23:25:19,441 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/.tmp/a/85d3d46c0a8b43f98dd3a30514edb425 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/a/85d3d46c0a8b43f98dd3a30514edb425 2024-11-26T23:25:19,450 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/a/85d3d46c0a8b43f98dd3a30514edb425, entries=2, sequenceid=2, filesize=5.0 K 2024-11-26T23:25:19,450 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 675c287fd51b34e2396cf85bae672631 in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-11-26T23:25:19,451 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/0000000000000000002 2024-11-26T23:25:19,452 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,452 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,455 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 675c287fd51b34e2396cf85bae672631 2024-11-26T23:25:19,458 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/675c287fd51b34e2396cf85bae672631/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-26T23:25:19,459 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 675c287fd51b34e2396cf85bae672631; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68573061, jitterRate=0.021818235516548157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:25:19,459 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 675c287fd51b34e2396cf85bae672631: Writing region info on filesystem at 1732663519392Initializing all the Stores at 1732663519394 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663519394Obtaining lock to block concurrent updates at 1732663519406 (+12 ms)Preparing flush snapshotting stores in 675c287fd51b34e2396cf85bae672631 at 1732663519406Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732663519406Flushing stores of testReplayEditsWrittenIntoWAL,,1732663519192.675c287fd51b34e2396cf85bae672631. at 1732663519406Flushing 675c287fd51b34e2396cf85bae672631/a: creating writer at 1732663519406Flushing 675c287fd51b34e2396cf85bae672631/a: appending metadata at 1732663519422 (+16 ms)Flushing 675c287fd51b34e2396cf85bae672631/a: closing flushed file at 1732663519422Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cd7c5f9: reopening flushed file at 1732663519440 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 675c287fd51b34e2396cf85bae672631 in 44ms, sequenceid=2, compaction requested=false; wal=null at 1732663519450 (+10 ms)Cleaning up temporary data from old regions at 1732663519452 (+2 ms)Region opened successfully at 1732663519459 (+7 ms) 2024-11-26T23:25:19,477 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=385 (was 375) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:39838 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:43798 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:43684 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:39744 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:46146 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 335), ProcessCount=11 (was 11), AvailableMemoryMB=7287 (was 7294) 2024-11-26T23:25:19,490 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=7287 2024-11-26T23:25:19,510 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:19,513 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:19,514 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:19,518 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33327347, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/hregion-33327347, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:19,537 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33327347/hregion-33327347.1732663519519, exclude list is [], retry=0 2024-11-26T23:25:19,541 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:19,541 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:19,541 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:19,544 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33327347/hregion-33327347.1732663519519 2024-11-26T23:25:19,544 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:19,545 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 29467d3ff4539a492870aefd22703957, NAME => 'testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/hbase 2024-11-26T23:25:19,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741854_1030 (size=64) 2024-11-26T23:25:19,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741854_1030 (size=64) 2024-11-26T23:25:19,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741854_1030 (size=64) 2024-11-26T23:25:19,556 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:19,558 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:19,560 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29467d3ff4539a492870aefd22703957 columnFamilyName a 2024-11-26T23:25:19,560 DEBUG [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:19,560 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(327): Store=29467d3ff4539a492870aefd22703957/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:19,560 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:19,562 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29467d3ff4539a492870aefd22703957 columnFamilyName b 2024-11-26T23:25:19,562 DEBUG [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:19,562 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(327): Store=29467d3ff4539a492870aefd22703957/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:19,563 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:19,564 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29467d3ff4539a492870aefd22703957 columnFamilyName c 2024-11-26T23:25:19,564 DEBUG [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:19,565 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(327): Store=29467d3ff4539a492870aefd22703957/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:19,565 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:19,566 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957 2024-11-26T23:25:19,566 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957 2024-11-26T23:25:19,568 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:19,568 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:19,568 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:19,570 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:19,572 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:19,572 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 29467d3ff4539a492870aefd22703957; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69483908, jitterRate=0.03539091348648071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:19,573 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 29467d3ff4539a492870aefd22703957: Writing region info on filesystem at 1732663519556Initializing all the Stores at 1732663519557 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663519558 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663519558Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663519558Cleaning up temporary data from old regions at 1732663519568 (+10 ms)Region opened successfully at 1732663519573 (+5 ms) 2024-11-26T23:25:19,573 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 29467d3ff4539a492870aefd22703957, disabling compactions & flushes 2024-11-26T23:25:19,573 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. 2024-11-26T23:25:19,573 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. 2024-11-26T23:25:19,573 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. after waiting 0 ms 2024-11-26T23:25:19,573 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. 2024-11-26T23:25:19,573 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. 2024-11-26T23:25:19,574 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 29467d3ff4539a492870aefd22703957: Waiting for close lock at 1732663519573Disabling compacts and flushes for region at 1732663519573Disabling writes for close at 1732663519573Writing region close event to WAL at 1732663519573Closed at 1732663519573 2024-11-26T23:25:19,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741853_1029 (size=93) 2024-11-26T23:25:19,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741853_1029 (size=93) 2024-11-26T23:25:19,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741853_1029 (size=93) 2024-11-26T23:25:19,579 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:19,579 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33327347:(num 1732663519519) 2024-11-26T23:25:19,580 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:19,582 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:19,596 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582, exclude list is [], retry=0 2024-11-26T23:25:19,599 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:19,599 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:19,599 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:19,602 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 2024-11-26T23:25:19,602 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:19,800 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582, size=0 (0bytes) 2024-11-26T23:25:19,800 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 might be still open, length is 0 2024-11-26T23:25:19,800 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 2024-11-26T23:25:19,801 WARN [IPC Server handler 3 on default port 45925 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-26T23:25:19,801 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 after 1ms 2024-11-26T23:25:20,675 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:39862 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36499:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39862 dst: /127.0.0.1:36499 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36499 remote=/127.0.0.1:39862]. Total timeout mills is 60000, 59093 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:20,676 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:46168 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37013:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46168 dst: /127.0.0.1:37013 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:20,677 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:43850 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43863:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43850 dst: /127.0.0.1:43863 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:20,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741855_1032 (size=470675) 2024-11-26T23:25:20,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741855_1032 (size=470675) 2024-11-26T23:25:20,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741855_1032 (size=470675) 2024-11-26T23:25:23,802 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 after 4002ms 2024-11-26T23:25:23,808 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 took 4009ms 2024-11-26T23:25:23,813 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732663519582.temp 2024-11-26T23:25:23,819 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000000001-wal.1732663519582.temp 2024-11-26T23:25:23,855 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:25:23,887 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582; continuing. 2024-11-26T23:25:23,887 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 so closing down 2024-11-26T23:25:23,887 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:23,888 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:23,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741856_1033 (size=470683) 2024-11-26T23:25:23,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741856_1033 (size=470683) 2024-11-26T23:25:23,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741856_1033 (size=470683) 2024-11-26T23:25:23,893 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000000001-wal.1732663519582.temp (wrote 3002 edits, skipped 0 edits in 29 ms) 2024-11-26T23:25:23,895 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000000001-wal.1732663519582.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000003002 2024-11-26T23:25:23,895 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 86 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T23:25:23,895 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582, journal: Splitting hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582, size=0 (0bytes) at 1732663519800Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000000001-wal.1732663519582.temp at 1732663523819 (+4019 ms)Split 1024 edits, skipped 0 edits. at 1732663523852 (+33 ms)Split 2048 edits, skipped 0 edits. at 1732663523874 (+22 ms)Finishing writing output for hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 so closing down at 1732663523887 (+13 ms)3 split writer threads finished at 1732663523888 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000000001-wal.1732663519582.temp (wrote 3002 edits, skipped 0 edits in 29 ms) at 1732663523893 (+5 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000000001-wal.1732663519582.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000003002 at 1732663523895 (+2 ms)Processed 3002 edits across 1 Regions in 86 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582, size=0, length=0, corrupted=false, cancelled=false at 1732663523895 2024-11-26T23:25:23,898 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 to hdfs://localhost:45925/hbase/oldWALs/wal.1732663519582 2024-11-26T23:25:23,899 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000003002 2024-11-26T23:25:23,900 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:23,902 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:23,916 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T23:25:23,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663523902, exclude list is [], retry=0 2024-11-26T23:25:23,919 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:23,919 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:23,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:23,922 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663523902 2024-11-26T23:25:23,923 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:23,923 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:23,925 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:23,926 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29467d3ff4539a492870aefd22703957 columnFamilyName a 2024-11-26T23:25:23,926 DEBUG [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:23,927 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(327): Store=29467d3ff4539a492870aefd22703957/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:23,927 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:23,928 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29467d3ff4539a492870aefd22703957 columnFamilyName b 2024-11-26T23:25:23,928 DEBUG [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:23,929 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(327): Store=29467d3ff4539a492870aefd22703957/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:23,929 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:23,930 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29467d3ff4539a492870aefd22703957 columnFamilyName c 2024-11-26T23:25:23,930 DEBUG [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:23,930 INFO [StoreOpener-29467d3ff4539a492870aefd22703957-1 {}] regionserver.HStore(327): Store=29467d3ff4539a492870aefd22703957/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:23,930 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:23,931 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957 2024-11-26T23:25:23,933 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957 2024-11-26T23:25:23,934 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000003002 2024-11-26T23:25:23,964 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T23:25:24,330 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 29467d3ff4539a492870aefd22703957 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T23:25:24,374 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/a/8bb6708872e746c4b02deafa93f6faaf is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732663519609/Put/seqid=0 2024-11-26T23:25:24,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741858_1035 (size=50463) 2024-11-26T23:25:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741858_1035 (size=50463) 2024-11-26T23:25:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741858_1035 (size=50463) 2024-11-26T23:25:24,383 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/a/8bb6708872e746c4b02deafa93f6faaf 2024-11-26T23:25:24,391 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/a/8bb6708872e746c4b02deafa93f6faaf as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/a/8bb6708872e746c4b02deafa93f6faaf 2024-11-26T23:25:24,398 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/a/8bb6708872e746c4b02deafa93f6faaf, entries=754, sequenceid=754, filesize=49.3 K 2024-11-26T23:25:24,399 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 29467d3ff4539a492870aefd22703957 in 69ms, sequenceid=754, compaction requested=false; wal=null 2024-11-26T23:25:24,414 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T23:25:24,414 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 29467d3ff4539a492870aefd22703957 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T23:25:24,424 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/a/2c39c23bd8734f5e9b367c9932402051 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732663519657/Put/seqid=0 2024-11-26T23:25:24,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741859_1036 (size=20072) 2024-11-26T23:25:24,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741859_1036 (size=20072) 2024-11-26T23:25:24,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741859_1036 (size=20072) 2024-11-26T23:25:24,437 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/a/2c39c23bd8734f5e9b367c9932402051 2024-11-26T23:25:24,466 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/b/977cbf6429ec4a1ea47a7d7d3458ddc4 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732663519676/Put/seqid=0 2024-11-26T23:25:24,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741860_1037 (size=35835) 2024-11-26T23:25:24,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741860_1037 (size=35835) 2024-11-26T23:25:24,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741860_1037 (size=35835) 2024-11-26T23:25:24,474 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/b/977cbf6429ec4a1ea47a7d7d3458ddc4 2024-11-26T23:25:24,481 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/a/2c39c23bd8734f5e9b367c9932402051 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/a/2c39c23bd8734f5e9b367c9932402051 2024-11-26T23:25:24,487 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/a/2c39c23bd8734f5e9b367c9932402051, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-26T23:25:24,489 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/b/977cbf6429ec4a1ea47a7d7d3458ddc4 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/b/977cbf6429ec4a1ea47a7d7d3458ddc4 2024-11-26T23:25:24,496 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/b/977cbf6429ec4a1ea47a7d7d3458ddc4, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-26T23:25:24,496 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 29467d3ff4539a492870aefd22703957 in 82ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-26T23:25:24,504 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T23:25:24,504 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 29467d3ff4539a492870aefd22703957 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-26T23:25:24,511 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/b/889e05ac090a4bb08d2f793fa9bf3538 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732663519694/Put/seqid=0 2024-11-26T23:25:24,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741861_1038 (size=35082) 2024-11-26T23:25:24,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741861_1038 (size=35082) 2024-11-26T23:25:24,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741861_1038 (size=35082) 2024-11-26T23:25:24,523 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/b/889e05ac090a4bb08d2f793fa9bf3538 2024-11-26T23:25:24,548 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/c/b147c48c7d514546a0e8a3cf3bf5d209 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732663519726/Put/seqid=0 2024-11-26T23:25:24,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741862_1039 (size=20825) 2024-11-26T23:25:24,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741862_1039 (size=20825) 2024-11-26T23:25:24,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741862_1039 (size=20825) 2024-11-26T23:25:24,557 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/c/b147c48c7d514546a0e8a3cf3bf5d209 2024-11-26T23:25:24,565 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/b/889e05ac090a4bb08d2f793fa9bf3538 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/b/889e05ac090a4bb08d2f793fa9bf3538 2024-11-26T23:25:24,571 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/b/889e05ac090a4bb08d2f793fa9bf3538, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-26T23:25:24,573 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/c/b147c48c7d514546a0e8a3cf3bf5d209 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/c/b147c48c7d514546a0e8a3cf3bf5d209 2024-11-26T23:25:24,579 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/c/b147c48c7d514546a0e8a3cf3bf5d209, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-26T23:25:24,579 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 29467d3ff4539a492870aefd22703957 in 75ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-26T23:25:24,585 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732663519766/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. 2024-11-26T23:25:24,587 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000003002 2024-11-26T23:25:24,588 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-26T23:25:24,588 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 29467d3ff4539a492870aefd22703957 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-26T23:25:24,596 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/c/92f3f18b3f404b9088e0e4f26d28652b is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732663519733/Put/seqid=0 2024-11-26T23:25:24,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741863_1040 (size=50301) 2024-11-26T23:25:24,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741863_1040 (size=50301) 2024-11-26T23:25:24,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741863_1040 (size=50301) 2024-11-26T23:25:24,608 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/c/92f3f18b3f404b9088e0e4f26d28652b 2024-11-26T23:25:24,614 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92f3f18b3f404b9088e0e4f26d28652b 2024-11-26T23:25:24,616 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/.tmp/c/92f3f18b3f404b9088e0e4f26d28652b as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/c/92f3f18b3f404b9088e0e4f26d28652b 2024-11-26T23:25:24,623 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 92f3f18b3f404b9088e0e4f26d28652b 2024-11-26T23:25:24,623 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/c/92f3f18b3f404b9088e0e4f26d28652b, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-26T23:25:24,623 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 29467d3ff4539a492870aefd22703957 in 35ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-26T23:25:24,624 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/0000000000000003002 2024-11-26T23:25:24,625 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:24,625 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:24,626 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:25:24,628 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 29467d3ff4539a492870aefd22703957 2024-11-26T23:25:24,630 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenIntoWAL/29467d3ff4539a492870aefd22703957/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-26T23:25:24,632 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 29467d3ff4539a492870aefd22703957; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66553087, jitterRate=-0.008281722664833069}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:25:24,632 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 29467d3ff4539a492870aefd22703957: Writing region info on filesystem at 1732663523923Initializing all the Stores at 1732663523925 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663523925Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663523925Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663523925Cleaning up temporary data from old regions at 1732663524625 (+700 ms)Region opened successfully at 1732663524632 (+7 ms) 2024-11-26T23:25:24,693 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 29467d3ff4539a492870aefd22703957, disabling compactions & flushes 2024-11-26T23:25:24,693 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. 2024-11-26T23:25:24,693 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. 2024-11-26T23:25:24,693 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. after waiting 0 ms 2024-11-26T23:25:24,693 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. 2024-11-26T23:25:24,696 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732663519511.29467d3ff4539a492870aefd22703957. 2024-11-26T23:25:24,696 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 29467d3ff4539a492870aefd22703957: Waiting for close lock at 1732663524693Disabling compacts and flushes for region at 1732663524693Disabling writes for close at 1732663524693Writing region close event to WAL at 1732663524696 (+3 ms)Closed at 1732663524696 2024-11-26T23:25:24,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741857_1034 (size=93) 2024-11-26T23:25:24,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741857_1034 (size=93) 2024-11-26T23:25:24,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741857_1034 (size=93) 2024-11-26T23:25:24,704 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:24,704 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732663523902) 2024-11-26T23:25:24,724 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=401 (was 385) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:46047 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1707855255_22 at /127.0.0.1:56280 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1707855255_22 at /127.0.0.1:36870 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:36207 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36207 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:45925 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1707855255_22 at /127.0.0.1:56288 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1707855255_22 at /127.0.0.1:50422 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:45925 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46047 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 332) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7220 (was 7287) 2024-11-26T23:25:24,737 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=401, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=345, ProcessCount=11, AvailableMemoryMB=7219 2024-11-26T23:25:24,753 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:24,755 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:24,755 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:24,758 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23457084, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/hregion-23457084, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:24,772 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23457084/hregion-23457084.1732663524759, exclude list is [], retry=0 2024-11-26T23:25:24,775 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:24,776 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:24,776 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:24,778 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23457084/hregion-23457084.1732663524759 2024-11-26T23:25:24,779 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:24,779 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 02c0564cbfd59ded78e8ef74cfd122b3, NAME => 'test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/hbase 2024-11-26T23:25:24,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741865_1042 (size=43) 2024-11-26T23:25:24,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741865_1042 (size=43) 2024-11-26T23:25:24,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741865_1042 (size=43) 2024-11-26T23:25:24,791 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:24,792 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:24,794 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02c0564cbfd59ded78e8ef74cfd122b3 columnFamilyName a 2024-11-26T23:25:24,794 DEBUG [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:24,795 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(327): Store=02c0564cbfd59ded78e8ef74cfd122b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:24,795 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:24,796 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02c0564cbfd59ded78e8ef74cfd122b3 columnFamilyName b 2024-11-26T23:25:24,796 DEBUG [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:24,797 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(327): Store=02c0564cbfd59ded78e8ef74cfd122b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:24,797 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:24,799 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02c0564cbfd59ded78e8ef74cfd122b3 columnFamilyName c 2024-11-26T23:25:24,799 DEBUG [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:24,799 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(327): Store=02c0564cbfd59ded78e8ef74cfd122b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:24,799 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:24,800 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:24,800 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:24,802 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:24,802 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:24,802 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:24,804 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:24,807 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:24,807 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 02c0564cbfd59ded78e8ef74cfd122b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69984690, jitterRate=0.04285314679145813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:24,808 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 02c0564cbfd59ded78e8ef74cfd122b3: Writing region info on filesystem at 1732663524791Initializing all the Stores at 1732663524792 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663524792Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663524792Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663524792Cleaning up temporary data from old regions at 1732663524802 (+10 ms)Region opened successfully at 1732663524808 (+6 ms) 2024-11-26T23:25:24,808 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 02c0564cbfd59ded78e8ef74cfd122b3, disabling compactions & flushes 2024-11-26T23:25:24,808 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. 2024-11-26T23:25:24,809 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. 2024-11-26T23:25:24,809 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. after waiting 0 ms 2024-11-26T23:25:24,809 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. 2024-11-26T23:25:24,809 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. 2024-11-26T23:25:24,809 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 02c0564cbfd59ded78e8ef74cfd122b3: Waiting for close lock at 1732663524808Disabling compacts and flushes for region at 1732663524808Disabling writes for close at 1732663524809 (+1 ms)Writing region close event to WAL at 1732663524809Closed at 1732663524809 2024-11-26T23:25:24,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741864_1041 (size=93) 2024-11-26T23:25:24,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741864_1041 (size=93) 2024-11-26T23:25:24,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741864_1041 (size=93) 2024-11-26T23:25:24,815 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:24,815 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23457084:(num 1732663524759) 2024-11-26T23:25:24,815 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:24,817 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:24,831 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818, exclude list is [], retry=0 2024-11-26T23:25:24,834 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:24,835 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:24,835 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:24,837 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818 2024-11-26T23:25:24,838 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:24,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741866_1043 (size=344306) 2024-11-26T23:25:24,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741866_1043 (size=344306) 2024-11-26T23:25:24,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741866_1043 (size=344306) 2024-11-26T23:25:25,001 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818, size=336.2 K (344306bytes) 2024-11-26T23:25:25,001 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818 2024-11-26T23:25:25,002 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818 after 1ms 2024-11-26T23:25:25,006 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818 took 5ms 2024-11-26T23:25:25,011 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732663524818.temp 2024-11-26T23:25:25,012 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000000001-wal.1732663524818.temp 2024-11-26T23:25:25,046 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818 so closing down 2024-11-26T23:25:25,046 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:25,046 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:25,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741867_1044 (size=344306) 2024-11-26T23:25:25,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741867_1044 (size=344306) 2024-11-26T23:25:25,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741867_1044 (size=344306) 2024-11-26T23:25:25,052 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000000001-wal.1732663524818.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-11-26T23:25:25,054 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000000001-wal.1732663524818.temp to hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003000 2024-11-26T23:25:25,054 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-26T23:25:25,054 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818, journal: Splitting hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818, size=336.2 K (344306bytes) at 1732663525001Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000000001-wal.1732663524818.temp at 1732663525012 (+11 ms)Split 1024 edits, skipped 0 edits. at 1732663525021 (+9 ms)Split 2048 edits, skipped 0 edits. at 1732663525034 (+13 ms)Finishing writing output for hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818 so closing down at 1732663525046 (+12 ms)3 split writer threads finished at 1732663525046Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000000001-wal.1732663524818.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1732663525052 (+6 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000000001-wal.1732663524818.temp to hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003000 at 1732663525054 (+2 ms)Processed 3000 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1732663525054 2024-11-26T23:25:25,056 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663524818 to hdfs://localhost:45925/hbase/oldWALs/wal.1732663524818 2024-11-26T23:25:25,057 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003000 2024-11-26T23:25:25,057 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:25,059 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:25,073 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059, exclude list is [], retry=0 2024-11-26T23:25:25,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:25,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:25,077 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:25,079 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059 2024-11-26T23:25:25,079 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:25,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741868_1045 (size=344433) 2024-11-26T23:25:25,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741868_1045 (size=344433) 2024-11-26T23:25:25,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741868_1045 (size=344433) 2024-11-26T23:25:25,245 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059, size=336.4 K (344433bytes) 2024-11-26T23:25:25,245 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059 2024-11-26T23:25:25,245 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059 after 0ms 2024-11-26T23:25:25,249 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059 took 5ms 2024-11-26T23:25:25,253 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732663525059.temp 2024-11-26T23:25:25,254 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003001-wal.1732663525059.temp 2024-11-26T23:25:25,282 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059 so closing down 2024-11-26T23:25:25,282 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:25,282 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:25,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741869_1046 (size=344433) 2024-11-26T23:25:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741869_1046 (size=344433) 2024-11-26T23:25:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741869_1046 (size=344433) 2024-11-26T23:25:25,287 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003001-wal.1732663525059.temp (wrote 3000 edits, skipped 0 edits in 12 ms) 2024-11-26T23:25:25,288 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003001-wal.1732663525059.temp to hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000006000 2024-11-26T23:25:25,288 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-26T23:25:25,288 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059, journal: Splitting hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059, size=336.4 K (344433bytes) at 1732663525245Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003001-wal.1732663525059.temp at 1732663525254 (+9 ms)Split 1024 edits, skipped 0 edits. at 1732663525262 (+8 ms)Split 2048 edits, skipped 0 edits. at 1732663525273 (+11 ms)Finishing writing output for hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059 so closing down at 1732663525282 (+9 ms)3 split writer threads finished at 1732663525282Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003001-wal.1732663525059.temp (wrote 3000 edits, skipped 0 edits in 12 ms) at 1732663525287 (+5 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003001-wal.1732663525059.temp to hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000006000 at 1732663525288 (+1 ms)Processed 3000 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1732663525288 2024-11-26T23:25:25,290 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525059 to hdfs://localhost:45925/hbase/oldWALs/wal.1732663525059 2024-11-26T23:25:25,291 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000006000 2024-11-26T23:25:25,291 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:25,293 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/test2727-manual,16010,1732663524752, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:25,308 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525294, exclude list is [], retry=0 2024-11-26T23:25:25,311 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:25,312 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:25,312 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:25,316 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732663524752/wal.1732663525294 2024-11-26T23:25:25,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:25,317 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 02c0564cbfd59ded78e8ef74cfd122b3, NAME => 'test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:25,317 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:25,317 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,318 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,319 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,320 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02c0564cbfd59ded78e8ef74cfd122b3 columnFamilyName a 2024-11-26T23:25:25,320 DEBUG [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:25,321 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(327): Store=02c0564cbfd59ded78e8ef74cfd122b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:25,321 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,322 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02c0564cbfd59ded78e8ef74cfd122b3 columnFamilyName b 2024-11-26T23:25:25,322 DEBUG [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:25,323 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(327): Store=02c0564cbfd59ded78e8ef74cfd122b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:25,323 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,324 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02c0564cbfd59ded78e8ef74cfd122b3 columnFamilyName c 2024-11-26T23:25:25,324 DEBUG [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:25,324 INFO [StoreOpener-02c0564cbfd59ded78e8ef74cfd122b3-1 {}] regionserver.HStore(327): Store=02c0564cbfd59ded78e8ef74cfd122b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:25,324 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,325 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,327 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,328 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003000 2024-11-26T23:25:25,369 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003000 2024-11-26T23:25:25,370 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000006000 2024-11-26T23:25:25,416 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000006000 2024-11-26T23:25:25,417 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 02c0564cbfd59ded78e8ef74cfd122b3 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-26T23:25:25,449 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/.tmp/a/482c1b406f89429fb501daf8a634b860 is 41, key is test2727/a:100/1732663525084/Put/seqid=0 2024-11-26T23:25:25,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741871_1048 (size=84227) 2024-11-26T23:25:25,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741871_1048 (size=84227) 2024-11-26T23:25:25,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741871_1048 (size=84227) 2024-11-26T23:25:25,457 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/.tmp/a/482c1b406f89429fb501daf8a634b860 2024-11-26T23:25:25,485 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/.tmp/b/81c7667ded2346dfa88d8ee7ab97ef4e is 41, key is test2727/b:100/1732663525134/Put/seqid=0 2024-11-26T23:25:25,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741872_1049 (size=84609) 2024-11-26T23:25:25,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741872_1049 (size=84609) 2024-11-26T23:25:25,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741872_1049 (size=84609) 2024-11-26T23:25:25,491 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/.tmp/b/81c7667ded2346dfa88d8ee7ab97ef4e 2024-11-26T23:25:25,519 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/.tmp/c/352e30be6e134fbaa8ddf9ce68984bb1 is 41, key is test2727/c:100/1732663525182/Put/seqid=0 2024-11-26T23:25:25,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741873_1050 (size=84609) 2024-11-26T23:25:25,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741873_1050 (size=84609) 2024-11-26T23:25:25,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741873_1050 (size=84609) 2024-11-26T23:25:25,527 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/.tmp/c/352e30be6e134fbaa8ddf9ce68984bb1 2024-11-26T23:25:25,535 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/.tmp/a/482c1b406f89429fb501daf8a634b860 as hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/a/482c1b406f89429fb501daf8a634b860 2024-11-26T23:25:25,542 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/a/482c1b406f89429fb501daf8a634b860, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-26T23:25:25,543 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/.tmp/b/81c7667ded2346dfa88d8ee7ab97ef4e as hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/b/81c7667ded2346dfa88d8ee7ab97ef4e 2024-11-26T23:25:25,551 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/b/81c7667ded2346dfa88d8ee7ab97ef4e, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-26T23:25:25,552 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/.tmp/c/352e30be6e134fbaa8ddf9ce68984bb1 as hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/c/352e30be6e134fbaa8ddf9ce68984bb1 2024-11-26T23:25:25,559 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/c/352e30be6e134fbaa8ddf9ce68984bb1, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-26T23:25:25,559 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 02c0564cbfd59ded78e8ef74cfd122b3 in 143ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-26T23:25:25,560 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000003000 2024-11-26T23:25:25,561 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/0000000000000006000 2024-11-26T23:25:25,562 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,562 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,563 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:25,564 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 02c0564cbfd59ded78e8ef74cfd122b3 2024-11-26T23:25:25,567 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/test2727/02c0564cbfd59ded78e8ef74cfd122b3/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-26T23:25:25,568 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 02c0564cbfd59ded78e8ef74cfd122b3; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72013596, jitterRate=0.0730862021446228}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:25,569 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 02c0564cbfd59ded78e8ef74cfd122b3: Writing region info on filesystem at 1732663525318Initializing all the Stores at 1732663525319 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663525319Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663525319Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663525319Obtaining lock to block concurrent updates at 1732663525417 (+98 ms)Preparing flush snapshotting stores in 02c0564cbfd59ded78e8ef74cfd122b3 at 1732663525417Finished memstore snapshotting test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732663525417Flushing stores of test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. at 1732663525417Flushing 02c0564cbfd59ded78e8ef74cfd122b3/a: creating writer at 1732663525417Flushing 02c0564cbfd59ded78e8ef74cfd122b3/a: appending metadata at 1732663525448 (+31 ms)Flushing 02c0564cbfd59ded78e8ef74cfd122b3/a: closing flushed file at 1732663525448Flushing 02c0564cbfd59ded78e8ef74cfd122b3/b: creating writer at 1732663525463 (+15 ms)Flushing 02c0564cbfd59ded78e8ef74cfd122b3/b: appending metadata at 1732663525483 (+20 ms)Flushing 02c0564cbfd59ded78e8ef74cfd122b3/b: closing flushed file at 1732663525483Flushing 02c0564cbfd59ded78e8ef74cfd122b3/c: creating writer at 1732663525498 (+15 ms)Flushing 02c0564cbfd59ded78e8ef74cfd122b3/c: appending metadata at 1732663525518 (+20 ms)Flushing 02c0564cbfd59ded78e8ef74cfd122b3/c: closing flushed file at 1732663525518Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fcfcb82: reopening flushed file at 1732663525534 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@110470f3: reopening flushed file at 1732663525542 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@588e6c94: reopening flushed file at 1732663525551 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 02c0564cbfd59ded78e8ef74cfd122b3 in 143ms, sequenceid=6000, compaction requested=false; wal=null at 1732663525559 (+8 ms)Cleaning up temporary data from old regions at 1732663525562 (+3 ms)Region opened successfully at 1732663525569 (+7 ms) 2024-11-26T23:25:25,571 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-26T23:25:25,571 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 02c0564cbfd59ded78e8ef74cfd122b3, disabling compactions & flushes 2024-11-26T23:25:25,571 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. 2024-11-26T23:25:25,571 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. 2024-11-26T23:25:25,571 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. after waiting 0 ms 2024-11-26T23:25:25,571 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. 2024-11-26T23:25:25,572 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732663524753.02c0564cbfd59ded78e8ef74cfd122b3. 2024-11-26T23:25:25,572 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 02c0564cbfd59ded78e8ef74cfd122b3: Waiting for close lock at 1732663525571Disabling compacts and flushes for region at 1732663525571Disabling writes for close at 1732663525571Writing region close event to WAL at 1732663525572 (+1 ms)Closed at 1732663525572 2024-11-26T23:25:25,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741870_1047 (size=93) 2024-11-26T23:25:25,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741870_1047 (size=93) 2024-11-26T23:25:25,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741870_1047 (size=93) 2024-11-26T23:25:25,580 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:25,580 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732663525294) 2024-11-26T23:25:25,596 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=404 (was 401) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:50526 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:36988 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 345), ProcessCount=11 (was 11), AvailableMemoryMB=7044 (was 7219) 2024-11-26T23:25:25,609 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=345, ProcessCount=11, AvailableMemoryMB=7043 2024-11-26T23:25:25,627 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:25,634 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:25,635 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732663525635 2024-11-26T23:25:25,643 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635 2024-11-26T23:25:25,645 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:25,646 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c799c2e16642507c6d526ba1433121bb, NAME => 'testSequentialEditLogSeqNum,,1732663525628.c799c2e16642507c6d526ba1433121bb.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:25,647 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732663525628.c799c2e16642507c6d526ba1433121bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:25,647 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c799c2e16642507c6d526ba1433121bb 2024-11-26T23:25:25,647 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c799c2e16642507c6d526ba1433121bb 2024-11-26T23:25:25,648 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb doesn't exist for region: c799c2e16642507c6d526ba1433121bb on table testSequentialEditLogSeqNum 2024-11-26T23:25:25,648 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: c799c2e16642507c6d526ba1433121bb on table testSequentialEditLogSeqNum 2024-11-26T23:25:25,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741875_1052 (size=62) 2024-11-26T23:25:25,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741875_1052 (size=62) 2024-11-26T23:25:25,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741875_1052 (size=62) 2024-11-26T23:25:25,659 INFO [StoreOpener-c799c2e16642507c6d526ba1433121bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c799c2e16642507c6d526ba1433121bb 2024-11-26T23:25:25,661 INFO [StoreOpener-c799c2e16642507c6d526ba1433121bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c799c2e16642507c6d526ba1433121bb columnFamilyName a 2024-11-26T23:25:25,661 DEBUG [StoreOpener-c799c2e16642507c6d526ba1433121bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:25,661 INFO [StoreOpener-c799c2e16642507c6d526ba1433121bb-1 {}] regionserver.HStore(327): Store=c799c2e16642507c6d526ba1433121bb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:25,661 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c799c2e16642507c6d526ba1433121bb 2024-11-26T23:25:25,662 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb 2024-11-26T23:25:25,663 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb 2024-11-26T23:25:25,663 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c799c2e16642507c6d526ba1433121bb 2024-11-26T23:25:25,663 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c799c2e16642507c6d526ba1433121bb 2024-11-26T23:25:25,665 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c799c2e16642507c6d526ba1433121bb 2024-11-26T23:25:25,667 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:25,667 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c799c2e16642507c6d526ba1433121bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74894936, jitterRate=0.11602151393890381}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:25:25,669 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c799c2e16642507c6d526ba1433121bb: Writing region info on filesystem at 1732663525647Initializing all the Stores at 1732663525659 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663525659Cleaning up temporary data from old regions at 1732663525663 (+4 ms)Region opened successfully at 1732663525669 (+6 ms) 2024-11-26T23:25:25,681 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c799c2e16642507c6d526ba1433121bb 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-26T23:25:25,703 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/.tmp/a/48c255369a53478c92f71e8c2674d3ff is 81, key is testSequentialEditLogSeqNum/a:x0/1732663525669/Put/seqid=0 2024-11-26T23:25:25,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741876_1053 (size=5833) 2024-11-26T23:25:25,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741876_1053 (size=5833) 2024-11-26T23:25:25,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741876_1053 (size=5833) 2024-11-26T23:25:25,711 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/.tmp/a/48c255369a53478c92f71e8c2674d3ff 2024-11-26T23:25:25,719 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/.tmp/a/48c255369a53478c92f71e8c2674d3ff as hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/a/48c255369a53478c92f71e8c2674d3ff 2024-11-26T23:25:25,726 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/a/48c255369a53478c92f71e8c2674d3ff, entries=10, sequenceid=13, filesize=5.7 K 2024-11-26T23:25:25,727 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for c799c2e16642507c6d526ba1433121bb in 47ms, sequenceid=13, compaction requested=false 2024-11-26T23:25:25,727 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c799c2e16642507c6d526ba1433121bb: 2024-11-26T23:25:25,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:25,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:25,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:25,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:25,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:25,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741874_1051 (size=3017) 2024-11-26T23:25:25,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741874_1051 (size=3017) 2024-11-26T23:25:25,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741874_1051 (size=3017) 2024-11-26T23:25:25,755 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635, size=2.9 K (3017bytes) 2024-11-26T23:25:25,755 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635 2024-11-26T23:25:25,756 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635 after 1ms 2024-11-26T23:25:25,759 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635 took 4ms 2024-11-26T23:25:25,761 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635 so closing down 2024-11-26T23:25:25,761 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:25,762 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732663525635.temp 2024-11-26T23:25:25,764 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/recovered.edits/0000000000000000003-wal.1732663525635.temp 2024-11-26T23:25:25,764 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:25,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741877_1054 (size=2433) 2024-11-26T23:25:25,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741877_1054 (size=2433) 2024-11-26T23:25:25,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741877_1054 (size=2433) 2024-11-26T23:25:25,773 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/recovered.edits/0000000000000000003-wal.1732663525635.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:25,775 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/recovered.edits/0000000000000000003-wal.1732663525635.temp to hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/recovered.edits/0000000000000000020 2024-11-26T23:25:25,775 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-26T23:25:25,775 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635, journal: Splitting hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635, size=2.9 K (3017bytes) at 1732663525755Finishing writing output for hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635 so closing down at 1732663525761 (+6 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/recovered.edits/0000000000000000003-wal.1732663525635.temp at 1732663525764 (+3 ms)3 split writer threads finished at 1732663525764Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/recovered.edits/0000000000000000003-wal.1732663525635.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732663525773 (+9 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/recovered.edits/0000000000000000003-wal.1732663525635.temp to hdfs://localhost:45925/hbase/data/default/testSequentialEditLogSeqNum/c799c2e16642507c6d526ba1433121bb/recovered.edits/0000000000000000020 at 1732663525775 (+2 ms)Processed 17 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:45925/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732663525627/wal.1732663525635, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1732663525775 2024-11-26T23:25:25,789 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:50526 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:36988 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 345), ProcessCount=11 (was 11), AvailableMemoryMB=7027 (was 7043) 2024-11-26T23:25:25,803 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=345, ProcessCount=11, AvailableMemoryMB=7023 2024-11-26T23:25:25,820 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:25,823 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:25,845 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:25,848 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77287442, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/hregion-77287442, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:25,862 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77287442/hregion-77287442.1732663525848, exclude list is [], retry=0 2024-11-26T23:25:25,866 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:25,866 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:25,866 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:25,869 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77287442/hregion-77287442.1732663525848 2024-11-26T23:25:25,870 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:25,870 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d4706ac63c1aff439656e2c520cf7bcf, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/hbase 2024-11-26T23:25:25,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741879_1056 (size=70) 2024-11-26T23:25:25,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741879_1056 (size=70) 2024-11-26T23:25:25,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741879_1056 (size=70) 2024-11-26T23:25:25,880 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:25,882 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,884 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4706ac63c1aff439656e2c520cf7bcf columnFamilyName a 2024-11-26T23:25:25,884 DEBUG [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:25,885 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(327): Store=d4706ac63c1aff439656e2c520cf7bcf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:25,885 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,887 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4706ac63c1aff439656e2c520cf7bcf columnFamilyName b 2024-11-26T23:25:25,887 DEBUG [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:25,888 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(327): Store=d4706ac63c1aff439656e2c520cf7bcf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:25,888 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,889 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4706ac63c1aff439656e2c520cf7bcf columnFamilyName c 2024-11-26T23:25:25,889 DEBUG [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:25,890 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(327): Store=d4706ac63c1aff439656e2c520cf7bcf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:25,890 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,891 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,891 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,892 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,893 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,893 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:25,894 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,897 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:25,898 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d4706ac63c1aff439656e2c520cf7bcf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73093156, jitterRate=0.08917289972305298}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:25,899 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d4706ac63c1aff439656e2c520cf7bcf: Writing region info on filesystem at 1732663525881Initializing all the Stores at 1732663525882 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663525882Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663525882Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663525882Cleaning up temporary data from old regions at 1732663525893 (+11 ms)Region opened successfully at 1732663525899 (+6 ms) 2024-11-26T23:25:25,899 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d4706ac63c1aff439656e2c520cf7bcf, disabling compactions & flushes 2024-11-26T23:25:25,899 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. 2024-11-26T23:25:25,899 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. 2024-11-26T23:25:25,899 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. after waiting 0 ms 2024-11-26T23:25:25,899 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. 2024-11-26T23:25:25,900 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. 2024-11-26T23:25:25,900 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d4706ac63c1aff439656e2c520cf7bcf: Waiting for close lock at 1732663525899Disabling compacts and flushes for region at 1732663525899Disabling writes for close at 1732663525899Writing region close event to WAL at 1732663525900 (+1 ms)Closed at 1732663525900 2024-11-26T23:25:25,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741878_1055 (size=93) 2024-11-26T23:25:25,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741878_1055 (size=93) 2024-11-26T23:25:25,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741878_1055 (size=93) 2024-11-26T23:25:25,907 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:25,907 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77287442:(num 1732663525848) 2024-11-26T23:25:25,907 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:25,910 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:25,933 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910, exclude list is [], retry=0 2024-11-26T23:25:25,936 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:25,936 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:25,937 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:25,939 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 2024-11-26T23:25:25,939 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:25,939 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d4706ac63c1aff439656e2c520cf7bcf, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:25,939 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:25,940 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,940 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,941 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,942 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4706ac63c1aff439656e2c520cf7bcf columnFamilyName a 2024-11-26T23:25:25,942 DEBUG [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:25,943 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(327): Store=d4706ac63c1aff439656e2c520cf7bcf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:25,943 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,944 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4706ac63c1aff439656e2c520cf7bcf columnFamilyName b 2024-11-26T23:25:25,944 DEBUG [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:25,944 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(327): Store=d4706ac63c1aff439656e2c520cf7bcf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:25,945 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,945 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4706ac63c1aff439656e2c520cf7bcf columnFamilyName c 2024-11-26T23:25:25,945 DEBUG [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:25,946 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(327): Store=d4706ac63c1aff439656e2c520cf7bcf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:25,946 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,947 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,948 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,949 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,949 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,950 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:25,951 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:25,952 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d4706ac63c1aff439656e2c520cf7bcf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67454999, jitterRate=0.0051578134298324585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:25,953 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d4706ac63c1aff439656e2c520cf7bcf: Writing region info on filesystem at 1732663525940Initializing all the Stores at 1732663525941 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663525941Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663525941Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663525941Cleaning up temporary data from old regions at 1732663525949 (+8 ms)Region opened successfully at 1732663525953 (+4 ms) 2024-11-26T23:25:25,958 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732663525957/Put/seqid=0 2024-11-26T23:25:25,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741881_1058 (size=4826) 2024-11-26T23:25:25,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741881_1058 (size=4826) 2024-11-26T23:25:25,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741881_1058 (size=4826) 2024-11-26T23:25:25,966 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45925/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in d4706ac63c1aff439656e2c520cf7bcf/a 2024-11-26T23:25:25,974 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-26T23:25:25,975 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T23:25:25,975 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d4706ac63c1aff439656e2c520cf7bcf: 2024-11-26T23:25:25,976 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/a/19643f62742b46ca804b54ab20abe04e_SeqId_3_ 2024-11-26T23:25:25,977 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45925/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into d4706ac63c1aff439656e2c520cf7bcf/a as hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/a/19643f62742b46ca804b54ab20abe04e_SeqId_3_ - updating store file list. 2024-11-26T23:25:25,983 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 19643f62742b46ca804b54ab20abe04e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-26T23:25:25,984 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/a/19643f62742b46ca804b54ab20abe04e_SeqId_3_ into d4706ac63c1aff439656e2c520cf7bcf/a 2024-11-26T23:25:25,984 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45925/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into d4706ac63c1aff439656e2c520cf7bcf/a (new location: hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/a/19643f62742b46ca804b54ab20abe04e_SeqId_3_) 2024-11-26T23:25:26,028 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910, size=0 (0bytes) 2024-11-26T23:25:26,028 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 might be still open, length is 0 2024-11-26T23:25:26,028 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 2024-11-26T23:25:26,028 WARN [IPC Server handler 3 on default port 45925 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-26T23:25:26,029 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 after 1ms 2024-11-26T23:25:26,168 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T23:25:26,168 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:26,171 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-26T23:25:26,171 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:26,172 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:25:26,172 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T23:25:26,173 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T23:25:26,173 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:26,174 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-26T23:25:26,174 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:26,176 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-26T23:25:26,176 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:26,670 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:37068 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37013:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37068 dst: /127.0.0.1:37013 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37013 remote=/127.0.0.1:37068]. Total timeout mills is 60000, 59320 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:26,670 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:56458 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43863:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56458 dst: /127.0.0.1:43863 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:26,670 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:50654 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36499:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50654 dst: /127.0.0.1:36499 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:26,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741880_1059 (size=504) 2024-11-26T23:25:26,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741880_1059 (size=504) 2024-11-26T23:25:26,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741880_1059 (size=504) 2024-11-26T23:25:30,030 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 after 4002ms 2024-11-26T23:25:30,033 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 took 4006ms 2024-11-26T23:25:30,035 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910; continuing. 2024-11-26T23:25:30,035 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 so closing down 2024-11-26T23:25:30,035 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:30,037 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732663525910.temp 2024-11-26T23:25:30,038 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005-wal.1732663525910.temp 2024-11-26T23:25:30,039 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:30,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741882_1060 (size=238) 2024-11-26T23:25:30,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741882_1060 (size=238) 2024-11-26T23:25:30,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741882_1060 (size=238) 2024-11-26T23:25:30,047 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005-wal.1732663525910.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:30,048 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005-wal.1732663525910.temp to hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005 2024-11-26T23:25:30,048 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 14 ms; skipped=1; WAL=hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T23:25:30,048 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910, journal: Splitting hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910, size=0 (0bytes) at 1732663526028Finishing writing output for hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 so closing down at 1732663530035 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005-wal.1732663525910.temp at 1732663530038 (+3 ms)3 split writer threads finished at 1732663530039 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005-wal.1732663525910.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732663530047 (+8 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005-wal.1732663525910.temp to hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005 at 1732663530048 (+1 ms)Processed 2 edits across 1 Regions in 14 ms; skipped=1; WAL=hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910, size=0, length=0, corrupted=false, cancelled=false at 1732663530048 2024-11-26T23:25:30,050 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 to hdfs://localhost:45925/hbase/oldWALs/wal.1732663525910 2024-11-26T23:25:30,051 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005 2024-11-26T23:25:30,051 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:30,053 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:30,091 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663530053, exclude list is [], retry=0 2024-11-26T23:25:30,094 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:30,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:30,095 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:30,097 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663530053 2024-11-26T23:25:30,097 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:30,097 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d4706ac63c1aff439656e2c520cf7bcf, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:30,098 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:30,098 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,098 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,100 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,101 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4706ac63c1aff439656e2c520cf7bcf columnFamilyName a 2024-11-26T23:25:30,101 DEBUG [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:30,108 DEBUG [StoreFileOpener-d4706ac63c1aff439656e2c520cf7bcf-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 19643f62742b46ca804b54ab20abe04e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-26T23:25:30,108 DEBUG [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/a/19643f62742b46ca804b54ab20abe04e_SeqId_3_ 2024-11-26T23:25:30,108 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(327): Store=d4706ac63c1aff439656e2c520cf7bcf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:30,108 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,109 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4706ac63c1aff439656e2c520cf7bcf columnFamilyName b 2024-11-26T23:25:30,109 DEBUG [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:30,110 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(327): Store=d4706ac63c1aff439656e2c520cf7bcf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:30,110 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,110 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4706ac63c1aff439656e2c520cf7bcf columnFamilyName c 2024-11-26T23:25:30,111 DEBUG [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:30,111 INFO [StoreOpener-d4706ac63c1aff439656e2c520cf7bcf-1 {}] regionserver.HStore(327): Store=d4706ac63c1aff439656e2c520cf7bcf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:30,111 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,112 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,113 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,114 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005 2024-11-26T23:25:30,116 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005 2024-11-26T23:25:30,116 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d4706ac63c1aff439656e2c520cf7bcf 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-26T23:25:30,131 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/.tmp/a/9b6a9633f4fb4cd48a953dbaffd5f582 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732663525989/Put/seqid=0 2024-11-26T23:25:30,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741884_1062 (size=5149) 2024-11-26T23:25:30,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741884_1062 (size=5149) 2024-11-26T23:25:30,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741884_1062 (size=5149) 2024-11-26T23:25:30,142 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/.tmp/a/9b6a9633f4fb4cd48a953dbaffd5f582 2024-11-26T23:25:30,147 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/.tmp/a/9b6a9633f4fb4cd48a953dbaffd5f582 as hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/a/9b6a9633f4fb4cd48a953dbaffd5f582 2024-11-26T23:25:30,153 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/a/9b6a9633f4fb4cd48a953dbaffd5f582, entries=1, sequenceid=5, filesize=5.0 K 2024-11-26T23:25:30,153 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for d4706ac63c1aff439656e2c520cf7bcf in 37ms, sequenceid=5, compaction requested=false; wal=null 2024-11-26T23:25:30,154 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/0000000000000000005 2024-11-26T23:25:30,155 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,155 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,156 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:30,157 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d4706ac63c1aff439656e2c520cf7bcf 2024-11-26T23:25:30,159 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d4706ac63c1aff439656e2c520cf7bcf/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-26T23:25:30,160 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d4706ac63c1aff439656e2c520cf7bcf; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60692452, jitterRate=-0.09561198949813843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:30,160 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d4706ac63c1aff439656e2c520cf7bcf: Writing region info on filesystem at 1732663530098Initializing all the Stores at 1732663530099 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663530099Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663530100 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663530100Obtaining lock to block concurrent updates at 1732663530116 (+16 ms)Preparing flush snapshotting stores in d4706ac63c1aff439656e2c520cf7bcf at 1732663530116Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732663530116Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. at 1732663530116Flushing d4706ac63c1aff439656e2c520cf7bcf/a: creating writer at 1732663530117 (+1 ms)Flushing d4706ac63c1aff439656e2c520cf7bcf/a: appending metadata at 1732663530130 (+13 ms)Flushing d4706ac63c1aff439656e2c520cf7bcf/a: closing flushed file at 1732663530131 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45638e27: reopening flushed file at 1732663530147 (+16 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for d4706ac63c1aff439656e2c520cf7bcf in 37ms, sequenceid=5, compaction requested=false; wal=null at 1732663530153 (+6 ms)Cleaning up temporary data from old regions at 1732663530155 (+2 ms)Region opened successfully at 1732663530160 (+5 ms) 2024-11-26T23:25:30,164 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d4706ac63c1aff439656e2c520cf7bcf, disabling compactions & flushes 2024-11-26T23:25:30,164 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. 2024-11-26T23:25:30,164 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. 2024-11-26T23:25:30,164 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. after waiting 0 ms 2024-11-26T23:25:30,164 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. 2024-11-26T23:25:30,165 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732663525821.d4706ac63c1aff439656e2c520cf7bcf. 2024-11-26T23:25:30,165 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d4706ac63c1aff439656e2c520cf7bcf: Waiting for close lock at 1732663530164Disabling compacts and flushes for region at 1732663530164Disabling writes for close at 1732663530164Writing region close event to WAL at 1732663530165 (+1 ms)Closed at 1732663530165 2024-11-26T23:25:30,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741883_1061 (size=93) 2024-11-26T23:25:30,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741883_1061 (size=93) 2024-11-26T23:25:30,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741883_1061 (size=93) 2024-11-26T23:25:30,170 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:30,170 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732663530053) 2024-11-26T23:25:30,184 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=412 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_5527718_22 at /127.0.0.1:50670 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_5527718_22 at /127.0.0.1:56490 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:45925 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:45925 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1024 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=318 (was 345), ProcessCount=11 (was 11), AvailableMemoryMB=6926 (was 7023) 2024-11-26T23:25:30,195 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=412, OpenFileDescriptor=1024, MaxFileDescriptor=1048576, SystemLoadAverage=318, ProcessCount=11, AvailableMemoryMB=6925 2024-11-26T23:25:30,211 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:30,216 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:25:30,220 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,35437,1732663515757 2024-11-26T23:25:30,222 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d0256bf 2024-11-26T23:25:30,223 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:25:30,225 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:25:30,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:25:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-26T23:25:30,237 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:25:30,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-26T23:25:30,239 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:30,241 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:25:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:25:30,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741885_1063 (size=694) 2024-11-26T23:25:30,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741885_1063 (size=694) 2024-11-26T23:25:30,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741885_1063 (size=694) 2024-11-26T23:25:30,253 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0071604d340e73f1eaa3d5a4b74f1f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e 2024-11-26T23:25:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741886_1064 (size=77) 2024-11-26T23:25:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741886_1064 (size=77) 2024-11-26T23:25:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741886_1064 (size=77) 2024-11-26T23:25:30,262 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:30,262 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing c0071604d340e73f1eaa3d5a4b74f1f4, disabling compactions & flushes 2024-11-26T23:25:30,262 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,263 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,263 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. after waiting 0 ms 2024-11-26T23:25:30,263 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,263 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,263 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0071604d340e73f1eaa3d5a4b74f1f4: Waiting for close lock at 1732663530262Disabling compacts and flushes for region at 1732663530262Disabling writes for close at 1732663530263 (+1 ms)Writing region close event to WAL at 1732663530263Closed at 1732663530263 2024-11-26T23:25:30,264 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:25:30,269 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732663530264"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732663530264"}]},"ts":"1732663530264"} 2024-11-26T23:25:30,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:25:30,274 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:25:30,276 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663530274"}]},"ts":"1732663530274"} 2024-11-26T23:25:30,280 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-26T23:25:30,281 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c707e9ace675=0} racks are {/default-rack=0} 2024-11-26T23:25:30,282 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T23:25:30,282 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T23:25:30,282 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T23:25:30,282 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T23:25:30,282 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T23:25:30,282 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T23:25:30,282 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T23:25:30,282 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T23:25:30,282 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T23:25:30,282 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T23:25:30,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, ASSIGN}] 2024-11-26T23:25:30,285 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, ASSIGN 2024-11-26T23:25:30,286 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, ASSIGN; state=OFFLINE, location=c707e9ace675,42261,1732663516429; forceNewPlan=false, retain=false 2024-11-26T23:25:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:25:30,438 INFO [c707e9ace675:35437 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T23:25:30,439 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=OPENING, regionLocation=c707e9ace675,42261,1732663516429 2024-11-26T23:25:30,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, ASSIGN because future has completed 2024-11-26T23:25:30,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,42261,1732663516429}] 2024-11-26T23:25:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:25:30,602 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,602 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c0071604d340e73f1eaa3d5a4b74f1f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:30,603 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,603 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:30,603 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,603 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,604 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,606 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf1 2024-11-26T23:25:30,606 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:30,606 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:30,606 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,608 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf2 2024-11-26T23:25:30,608 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:30,608 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:30,608 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,609 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,610 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,611 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,611 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,612 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T23:25:30,613 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,615 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:30,615 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c0071604d340e73f1eaa3d5a4b74f1f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67970840, jitterRate=0.012844443321228027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T23:25:30,616 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,616 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c0071604d340e73f1eaa3d5a4b74f1f4: Running coprocessor pre-open hook at 1732663530603Writing region info on filesystem at 1732663530603Initializing all the Stores at 1732663530604 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663530604Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663530604Cleaning up temporary data from old regions at 1732663530611 (+7 ms)Running coprocessor post-open hooks at 1732663530616 (+5 ms)Region opened successfully at 1732663530616 2024-11-26T23:25:30,618 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., pid=6, masterSystemTime=1732663530597 2024-11-26T23:25:30,620 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,620 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,621 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,42261,1732663516429 2024-11-26T23:25:30,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,42261,1732663516429 because future has completed 2024-11-26T23:25:30,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T23:25:30,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,42261,1732663516429 in 182 msec 2024-11-26T23:25:30,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T23:25:30,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, ASSIGN in 345 msec 2024-11-26T23:25:30,632 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:25:30,633 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663530632"}]},"ts":"1732663530632"} 2024-11-26T23:25:30,635 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-26T23:25:30,636 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:25:30,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 406 msec 2024-11-26T23:25:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:25:30,877 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-26T23:25:30,877 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-26T23:25:30,878 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T23:25:30,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-26T23:25:30,884 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T23:25:30,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-26T23:25:30,896 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=2] 2024-11-26T23:25:30,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=c0071604d340e73f1eaa3d5a4b74f1f4, source=c707e9ace675,42261,1732663516429, destination=c707e9ace675,34663,1732663516528, warming up region on c707e9ace675,34663,1732663516528 2024-11-26T23:25:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:25:30,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=c0071604d340e73f1eaa3d5a4b74f1f4, source=c707e9ace675,42261,1732663516429, destination=c707e9ace675,34663,1732663516528, running balancer 2024-11-26T23:25:30,917 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54629, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:25:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE 2024-11-26T23:25:30,918 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE 2024-11-26T23:25:30,920 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=CLOSING, regionLocation=c707e9ace675,42261,1732663516429 2024-11-26T23:25:30,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] regionserver.HRegion(7855): Warmup {ENCODED => c0071604d340e73f1eaa3d5a4b74f1f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:30,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:30,924 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35437 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=c707e9ace675,42261,1732663516429, table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T23:25:30,924 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE because future has completed 2024-11-26T23:25:30,925 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:25:30,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,42261,1732663516429}] 2024-11-26T23:25:30,925 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf1 2024-11-26T23:25:30,926 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:30,926 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:30,927 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:30,928 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf2 2024-11-26T23:25:30,928 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:30,928 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] regionserver.HRegion(1722): Closing c0071604d340e73f1eaa3d5a4b74f1f4, disabling compactions & flushes 2024-11-26T23:25:30,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. after waiting 0 ms 2024-11-26T23:25:30,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] regionserver.HRegion(1676): Region close journal for c0071604d340e73f1eaa3d5a4b74f1f4: Waiting for close lock at 1732663530929Disabling compacts and flushes for region at 1732663530929Disabling writes for close at 1732663530929Writing region close event to WAL at 1732663530929Closed at 1732663530929 2024-11-26T23:25:30,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-26T23:25:31,086 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,087 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:25:31,087 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing c0071604d340e73f1eaa3d5a4b74f1f4, disabling compactions & flushes 2024-11-26T23:25:31,087 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,087 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,087 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. after waiting 0 ms 2024-11-26T23:25:31,087 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,087 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing c0071604d340e73f1eaa3d5a4b74f1f4 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-26T23:25:31,105 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf1/66ef1193740149bb9ce63725fdaeb99e is 35, key is r1/cf1:q/1732663530898/Put/seqid=0 2024-11-26T23:25:31,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741887_1065 (size=4783) 2024-11-26T23:25:31,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741887_1065 (size=4783) 2024-11-26T23:25:31,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741887_1065 (size=4783) 2024-11-26T23:25:31,116 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf1/66ef1193740149bb9ce63725fdaeb99e 2024-11-26T23:25:31,124 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf1/66ef1193740149bb9ce63725fdaeb99e as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/66ef1193740149bb9ce63725fdaeb99e 2024-11-26T23:25:31,133 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/66ef1193740149bb9ce63725fdaeb99e, entries=1, sequenceid=5, filesize=4.7 K 2024-11-26T23:25:31,134 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for c0071604d340e73f1eaa3d5a4b74f1f4 in 47ms, sequenceid=5, compaction requested=false 2024-11-26T23:25:31,134 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-26T23:25:31,139 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-26T23:25:31,142 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,142 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for c0071604d340e73f1eaa3d5a4b74f1f4: Waiting for close lock at 1732663531087Running coprocessor pre-close hooks at 1732663531087Disabling compacts and flushes for region at 1732663531087Disabling writes for close at 1732663531087Obtaining lock to block concurrent updates at 1732663531087Preparing flush snapshotting stores in c0071604d340e73f1eaa3d5a4b74f1f4 at 1732663531087Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732663531088 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. at 1732663531089 (+1 ms)Flushing c0071604d340e73f1eaa3d5a4b74f1f4/cf1: creating writer at 1732663531089Flushing c0071604d340e73f1eaa3d5a4b74f1f4/cf1: appending metadata at 1732663531105 (+16 ms)Flushing c0071604d340e73f1eaa3d5a4b74f1f4/cf1: closing flushed file at 1732663531105Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27453516: reopening flushed file at 1732663531123 (+18 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for c0071604d340e73f1eaa3d5a4b74f1f4 in 47ms, sequenceid=5, compaction requested=false at 1732663531134 (+11 ms)Writing region close event to WAL at 1732663531135 (+1 ms)Running coprocessor post-close hooks at 1732663531140 (+5 ms)Closed at 1732663531142 (+2 ms) 2024-11-26T23:25:31,142 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding c0071604d340e73f1eaa3d5a4b74f1f4 move to c707e9ace675,34663,1732663516528 record at close sequenceid=5 2024-11-26T23:25:31,146 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,146 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=CLOSED 2024-11-26T23:25:31,149 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,42261,1732663516429 because future has completed 2024-11-26T23:25:31,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T23:25:31,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,42261,1732663516429 in 226 msec 2024-11-26T23:25:31,155 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE; state=CLOSED, location=c707e9ace675,34663,1732663516528; forceNewPlan=false, retain=false 2024-11-26T23:25:31,306 INFO [c707e9ace675:35437 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T23:25:31,306 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=OPENING, regionLocation=c707e9ace675,34663,1732663516528 2024-11-26T23:25:31,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE because future has completed 2024-11-26T23:25:31,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,34663,1732663516528}] 2024-11-26T23:25:31,468 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,468 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => c0071604d340e73f1eaa3d5a4b74f1f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:31,469 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,469 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:31,469 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,469 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,474 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,476 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf1 2024-11-26T23:25:31,476 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:31,482 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/66ef1193740149bb9ce63725fdaeb99e 2024-11-26T23:25:31,482 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:31,483 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,483 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf2 2024-11-26T23:25:31,484 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:31,484 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:31,484 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,485 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,486 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,487 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,487 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,488 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T23:25:31,489 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,490 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened c0071604d340e73f1eaa3d5a4b74f1f4; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61478143, jitterRate=-0.08390428125858307}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T23:25:31,490 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,491 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for c0071604d340e73f1eaa3d5a4b74f1f4: Running coprocessor pre-open hook at 1732663531469Writing region info on filesystem at 1732663531469Initializing all the Stores at 1732663531470 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663531470Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663531474 (+4 ms)Cleaning up temporary data from old regions at 1732663531487 (+13 ms)Running coprocessor post-open hooks at 1732663531491 (+4 ms)Region opened successfully at 1732663531491 2024-11-26T23:25:31,492 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., pid=9, masterSystemTime=1732663531463 2024-11-26T23:25:31,495 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,495 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=OPEN, openSeqNum=9, regionLocation=c707e9ace675,34663,1732663516528 2024-11-26T23:25:31,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,34663,1732663516528 because future has completed 2024-11-26T23:25:31,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-26T23:25:31,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,34663,1732663516528 in 192 msec 2024-11-26T23:25:31,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE in 589 msec 2024-11-26T23:25:31,524 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:25:31,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:25:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:33062 deadline: 1732663591530, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c707e9ace675 port=34663 startCode=1732663516528. As of locationSeqNum=5. 2024-11-26T23:25:31,565 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c707e9ace675 port=34663 startCode=1732663516528. As of locationSeqNum=5. 2024-11-26T23:25:31,566 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c707e9ace675 port=34663 startCode=1732663516528. As of locationSeqNum=5. 2024-11-26T23:25:31,566 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,34663,1732663516528, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c707e9ace675 port=34663 startCode=1732663516528. As of locationSeqNum=5. 2024-11-26T23:25:31,675 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:25:31,678 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:25:31,689 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c0071604d340e73f1eaa3d5a4b74f1f4 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-26T23:25:31,714 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf1/f2d463e262594580b69da9ef78ac4dfb is 29, key is r1/cf1:/1732663531679/DeleteFamily/seqid=0 2024-11-26T23:25:31,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741888_1066 (size=4906) 2024-11-26T23:25:31,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741888_1066 (size=4906) 2024-11-26T23:25:31,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741888_1066 (size=4906) 2024-11-26T23:25:31,724 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf1/f2d463e262594580b69da9ef78ac4dfb 2024-11-26T23:25:31,731 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f2d463e262594580b69da9ef78ac4dfb 2024-11-26T23:25:31,753 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf2/091e41faa1d2412fa57660cc0560f969 is 29, key is r1/cf2:/1732663531679/DeleteFamily/seqid=0 2024-11-26T23:25:31,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741889_1067 (size=4906) 2024-11-26T23:25:31,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741889_1067 (size=4906) 2024-11-26T23:25:31,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741889_1067 (size=4906) 2024-11-26T23:25:31,763 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf2/091e41faa1d2412fa57660cc0560f969 2024-11-26T23:25:31,770 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 091e41faa1d2412fa57660cc0560f969 2024-11-26T23:25:31,771 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf1/f2d463e262594580b69da9ef78ac4dfb as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/f2d463e262594580b69da9ef78ac4dfb 2024-11-26T23:25:31,779 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f2d463e262594580b69da9ef78ac4dfb 2024-11-26T23:25:31,779 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/f2d463e262594580b69da9ef78ac4dfb, entries=1, sequenceid=12, filesize=4.8 K 2024-11-26T23:25:31,780 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf2/091e41faa1d2412fa57660cc0560f969 as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/091e41faa1d2412fa57660cc0560f969 2024-11-26T23:25:31,787 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 091e41faa1d2412fa57660cc0560f969 2024-11-26T23:25:31,787 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/091e41faa1d2412fa57660cc0560f969, entries=1, sequenceid=12, filesize=4.8 K 2024-11-26T23:25:31,788 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for c0071604d340e73f1eaa3d5a4b74f1f4 in 99ms, sequenceid=12, compaction requested=false 2024-11-26T23:25:31,788 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c0071604d340e73f1eaa3d5a4b74f1f4: 2024-11-26T23:25:31,792 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-26T23:25:31,794 DEBUG [Time-limited test {}] regionserver.HStore(1541): c0071604d340e73f1eaa3d5a4b74f1f4/cf1 is initiating major compaction (all files) 2024-11-26T23:25:31,794 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:25:31,794 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:31,794 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c0071604d340e73f1eaa3d5a4b74f1f4/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,795 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/66ef1193740149bb9ce63725fdaeb99e, hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/f2d463e262594580b69da9ef78ac4dfb] into tmpdir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp, totalSize=9.5 K 2024-11-26T23:25:31,796 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 66ef1193740149bb9ce63725fdaeb99e, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732663530898 2024-11-26T23:25:31,796 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f2d463e262594580b69da9ef78ac4dfb, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-26T23:25:31,808 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): c0071604d340e73f1eaa3d5a4b74f1f4#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:25:31,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741890_1068 (size=4626) 2024-11-26T23:25:31,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741890_1068 (size=4626) 2024-11-26T23:25:31,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741890_1068 (size=4626) 2024-11-26T23:25:31,825 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf1/a8cdce13a493427c808d8c5c9f337b25 as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/a8cdce13a493427c808d8c5c9f337b25 2024-11-26T23:25:31,840 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in c0071604d340e73f1eaa3d5a4b74f1f4/cf1 of c0071604d340e73f1eaa3d5a4b74f1f4 into a8cdce13a493427c808d8c5c9f337b25(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:25:31,840 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c0071604d340e73f1eaa3d5a4b74f1f4: 2024-11-26T23:25:31,840 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-26T23:25:31,840 DEBUG [Time-limited test {}] regionserver.HStore(1541): c0071604d340e73f1eaa3d5a4b74f1f4/cf2 is initiating major compaction (all files) 2024-11-26T23:25:31,841 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:25:31,841 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:25:31,841 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c0071604d340e73f1eaa3d5a4b74f1f4/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,841 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/091e41faa1d2412fa57660cc0560f969] into tmpdir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp, totalSize=4.8 K 2024-11-26T23:25:31,842 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 091e41faa1d2412fa57660cc0560f969, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-26T23:25:31,848 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): c0071604d340e73f1eaa3d5a4b74f1f4#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:25:31,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741891_1069 (size=4592) 2024-11-26T23:25:31,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741891_1069 (size=4592) 2024-11-26T23:25:31,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741891_1069 (size=4592) 2024-11-26T23:25:31,865 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/.tmp/cf2/6a1b00d3e18c488cb0588ccea9fc9a42 as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/6a1b00d3e18c488cb0588ccea9fc9a42 2024-11-26T23:25:31,873 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in c0071604d340e73f1eaa3d5a4b74f1f4/cf2 of c0071604d340e73f1eaa3d5a4b74f1f4 into 6a1b00d3e18c488cb0588ccea9fc9a42(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:25:31,873 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c0071604d340e73f1eaa3d5a4b74f1f4: 2024-11-26T23:25:31,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=c0071604d340e73f1eaa3d5a4b74f1f4, source=c707e9ace675,34663,1732663516528, destination=c707e9ace675,42261,1732663516429, warming up region on c707e9ace675,42261,1732663516429 2024-11-26T23:25:31,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=c0071604d340e73f1eaa3d5a4b74f1f4, source=c707e9ace675,34663,1732663516528, destination=c707e9ace675,42261,1732663516429, running balancer 2024-11-26T23:25:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE 2024-11-26T23:25:31,880 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE 2024-11-26T23:25:31,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] regionserver.HRegion(7855): Warmup {ENCODED => c0071604d340e73f1eaa3d5a4b74f1f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:31,882 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=CLOSING, regionLocation=c707e9ace675,34663,1732663516528 2024-11-26T23:25:31,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:31,883 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,884 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf1 2024-11-26T23:25:31,884 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:31,885 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE because future has completed 2024-11-26T23:25:31,885 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-26T23:25:31,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,34663,1732663516528}] 2024-11-26T23:25:31,893 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/66ef1193740149bb9ce63725fdaeb99e 2024-11-26T23:25:31,898 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/a8cdce13a493427c808d8c5c9f337b25 2024-11-26T23:25:31,903 INFO [StoreFileOpener-c0071604d340e73f1eaa3d5a4b74f1f4-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f2d463e262594580b69da9ef78ac4dfb 2024-11-26T23:25:31,903 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/f2d463e262594580b69da9ef78ac4dfb 2024-11-26T23:25:31,903 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:31,903 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:31,905 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf2 2024-11-26T23:25:31,905 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:31,911 INFO [StoreFileOpener-c0071604d340e73f1eaa3d5a4b74f1f4-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 091e41faa1d2412fa57660cc0560f969 2024-11-26T23:25:31,912 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/091e41faa1d2412fa57660cc0560f969 2024-11-26T23:25:31,917 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/6a1b00d3e18c488cb0588ccea9fc9a42 2024-11-26T23:25:31,917 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] regionserver.HRegion(1722): Closing c0071604d340e73f1eaa3d5a4b74f1f4, disabling compactions & flushes 2024-11-26T23:25:31,917 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. after waiting 0 ms 2024-11-26T23:25:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42261 {}] regionserver.HRegion(1676): Region close journal for c0071604d340e73f1eaa3d5a4b74f1f4: Waiting for close lock at 1732663531917Disabling compacts and flushes for region at 1732663531917Disabling writes for close at 1732663531917Writing region close event to WAL at 1732663531918 (+1 ms)Closed at 1732663531918 2024-11-26T23:25:31,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-26T23:25:32,038 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,038 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-26T23:25:32,038 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing c0071604d340e73f1eaa3d5a4b74f1f4, disabling compactions & flushes 2024-11-26T23:25:32,038 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,038 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,039 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. after waiting 0 ms 2024-11-26T23:25:32,039 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,039 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/66ef1193740149bb9ce63725fdaeb99e, hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/f2d463e262594580b69da9ef78ac4dfb] to archive 2024-11-26T23:25:32,042 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:25:32,046 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/66ef1193740149bb9ce63725fdaeb99e to hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/66ef1193740149bb9ce63725fdaeb99e 2024-11-26T23:25:32,047 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/f2d463e262594580b69da9ef78ac4dfb to hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/f2d463e262594580b69da9ef78ac4dfb 2024-11-26T23:25:32,060 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/091e41faa1d2412fa57660cc0560f969] to archive 2024-11-26T23:25:32,061 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:25:32,063 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/091e41faa1d2412fa57660cc0560f969 to hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/091e41faa1d2412fa57660cc0560f969 2024-11-26T23:25:32,068 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-26T23:25:32,069 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,069 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for c0071604d340e73f1eaa3d5a4b74f1f4: Waiting for close lock at 1732663532038Running coprocessor pre-close hooks at 1732663532038Disabling compacts and flushes for region at 1732663532038Disabling writes for close at 1732663532039 (+1 ms)Writing region close event to WAL at 1732663532064 (+25 ms)Running coprocessor post-close hooks at 1732663532069 (+5 ms)Closed at 1732663532069 2024-11-26T23:25:32,069 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding c0071604d340e73f1eaa3d5a4b74f1f4 move to c707e9ace675,42261,1732663516429 record at close sequenceid=12 2024-11-26T23:25:32,072 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,073 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=CLOSED 2024-11-26T23:25:32,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,34663,1732663516528 because future has completed 2024-11-26T23:25:32,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-26T23:25:32,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,34663,1732663516528 in 192 msec 2024-11-26T23:25:32,081 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE; state=CLOSED, location=c707e9ace675,42261,1732663516429; forceNewPlan=false, retain=false 2024-11-26T23:25:32,232 INFO [c707e9ace675:35437 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T23:25:32,232 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=OPENING, regionLocation=c707e9ace675,42261,1732663516429 2024-11-26T23:25:32,235 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE because future has completed 2024-11-26T23:25:32,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,42261,1732663516429}] 2024-11-26T23:25:32,392 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,392 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => c0071604d340e73f1eaa3d5a4b74f1f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:32,392 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,393 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:32,393 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,393 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,395 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,395 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf1 2024-11-26T23:25:32,396 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:32,401 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/a8cdce13a493427c808d8c5c9f337b25 2024-11-26T23:25:32,401 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:32,402 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,403 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf2 2024-11-26T23:25:32,403 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:32,408 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/6a1b00d3e18c488cb0588ccea9fc9a42 2024-11-26T23:25:32,408 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:32,408 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,409 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,410 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,411 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,411 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,412 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T23:25:32,413 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,414 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened c0071604d340e73f1eaa3d5a4b74f1f4; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59694981, jitterRate=-0.11047546565532684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T23:25:32,414 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,415 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for c0071604d340e73f1eaa3d5a4b74f1f4: Running coprocessor pre-open hook at 1732663532393Writing region info on filesystem at 1732663532393Initializing all the Stores at 1732663532394 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663532394Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663532394Cleaning up temporary data from old regions at 1732663532411 (+17 ms)Running coprocessor post-open hooks at 1732663532414 (+3 ms)Region opened successfully at 1732663532415 (+1 ms) 2024-11-26T23:25:32,416 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., pid=12, masterSystemTime=1732663532388 2024-11-26T23:25:32,419 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,419 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,420 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=OPEN, openSeqNum=18, regionLocation=c707e9ace675,42261,1732663516429 2024-11-26T23:25:32,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,42261,1732663516429 because future has completed 2024-11-26T23:25:32,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-26T23:25:32,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,42261,1732663516429 in 188 msec 2024-11-26T23:25:32,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, REOPEN/MOVE in 547 msec 2024-11-26T23:25:32,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:25:32,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:25:32,435 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server c707e9ace675,42261,1732663516429: testing ***** 2024-11-26T23:25:32,435 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-26T23:25:32,437 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-26T23:25:32,438 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-26T23:25:32,440 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-26T23:25:32,441 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-26T23:25:32,447 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 274106760 }, "NonHeapMemoryUsage": { "committed": 170131456, "init": 7667712, "max": -1, "used": 167499680 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "c707e9ace675", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 13, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 4, "ProcessCallTime_median": 7, "ProcessCallTime_75th_percentile": 10, "ProcessCallTime_90th_percentile": 11, "ProcessCallTime_95th_percentile": 12, "ProcessCallTime_98th_percentile": 12, "ProcessCallTime_99th_percentile": 12, "ProcessCallTime_99.9th_percentile": 12, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 13, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 7, "TotalCallTime_75th_percentile": 10, "TotalCallTime_90th_percentile": 11, "TotalCallTime_95th_percentile": 12, "TotalCallTime_98th_percentile": 12, "TotalCallTime_99th_percentile": 12, "TotalCallTime_99.9th_percentile": 12, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "c707e9ace675", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:51163", "tag.serverName": "c707e9ace675,46791,1732663516570", "tag.clusterId": "92411619-9446-40ba-872e-d0d817f07ae3", "tag.Context": "regionserver", "tag.Hostname": "c707e9ace675", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732663516570, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-26T23:25:32,450 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35437 {}] master.MasterRpcServices(700): c707e9ace675,42261,1732663516429 reported a fatal error: ***** ABORTING region server c707e9ace675,42261,1732663516429: testing ***** 2024-11-26T23:25:32,452 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,42261,1732663516429' ***** 2024-11-26T23:25:32,452 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-26T23:25:32,452 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:25:32,452 INFO [RS:0;c707e9ace675:42261 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-26T23:25:32,452 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:25:32,452 INFO [RS:0;c707e9ace675:42261 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-26T23:25:32,453 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(3091): Received CLOSE for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34663 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:47972 deadline: 1732663592453, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c707e9ace675 port=42261 startCode=1732663516429. As of locationSeqNum=12. 2024-11-26T23:25:32,453 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(956): aborting server c707e9ace675,42261,1732663516429 2024-11-26T23:25:32,453 INFO [RS:0;c707e9ace675:42261 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:25:32,453 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,34663,1732663516528, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,34663,1732663516528, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c707e9ace675 port=42261 startCode=1732663516429. As of locationSeqNum=12. 2024-11-26T23:25:32,453 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c0071604d340e73f1eaa3d5a4b74f1f4, disabling compactions & flushes 2024-11-26T23:25:32,454 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,34663,1732663516528, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c707e9ace675 port=42261 startCode=1732663516429. As of locationSeqNum=12. 2024-11-26T23:25:32,454 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,454 INFO [RS:0;c707e9ace675:42261 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c707e9ace675:42261. 2024-11-26T23:25:32,454 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,454 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,34663,1732663516528, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c707e9ace675 port=42261 startCode=1732663516429. As of locationSeqNum=12. 2024-11-26T23:25:32,454 DEBUG [RS:0;c707e9ace675:42261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:25:32,454 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. after waiting 0 ms 2024-11-26T23:25:32,454 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,454 DEBUG [RS:0;c707e9ace675:42261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:25:32,454 INFO [RS:0;c707e9ace675:42261 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:25:32,454 INFO [RS:0;c707e9ace675:42261 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:25:32,454 INFO [RS:0;c707e9ace675:42261 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:25:32,454 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T23:25:32,455 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T23:25:32,455 DEBUG [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, c0071604d340e73f1eaa3d5a4b74f1f4=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.} 2024-11-26T23:25:32,455 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:25:32,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c707e9ace675,42261,1732663516429 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:32,455 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:25:32,455 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:25:32,455 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,455 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:25:32,456 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:25:32,456 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c0071604d340e73f1eaa3d5a4b74f1f4: Waiting for close lock at 1732663532453Running coprocessor pre-close hooks at 1732663532453Disabling compacts and flushes for region at 1732663532453Disabling writes for close at 1732663532454 (+1 ms)Writing region close event to WAL at 1732663532455 (+1 ms)Running coprocessor post-close hooks at 1732663532455Closed at 1732663532455 2024-11-26T23:25:32,456 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:32,456 DEBUG [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:25:32,456 ERROR [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-26T23:25:32,456 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:25:32,457 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c707e9ace675,42261,1732663516429 aborting 2024-11-26T23:25:32,457 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:25:32,457 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c707e9ace675,42261,1732663516429 aborting 2024-11-26T23:25:32,457 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663532455Running coprocessor pre-close hooks at 1732663532455Disabling compacts and flushes for region at 1732663532455Disabling writes for close at 1732663532455Writing region close event to WAL at 1732663532456 (+1 ms)Running coprocessor post-close hooks at 1732663532456Closed at 1732663532457 (+1 ms) 2024-11-26T23:25:32,457 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 from cache 2024-11-26T23:25:32,457 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T23:25:32,536 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:25:32,565 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:25:32,566 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1] 2024-11-26T23:25:32,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c707e9ace675,42261,1732663516429 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:32,568 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c707e9ace675,42261,1732663516429 aborting 2024-11-26T23:25:32,568 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c707e9ace675,42261,1732663516429 aborting 2024-11-26T23:25:32,568 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 from cache 2024-11-26T23:25:32,656 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,42261,1732663516429; all regions closed. 2024-11-26T23:25:32,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741836_1012 (size=6845) 2024-11-26T23:25:32,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741836_1012 (size=6845) 2024-11-26T23:25:32,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741836_1012 (size=6845) 2024-11-26T23:25:32,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741833_1009 (size=2142) 2024-11-26T23:25:32,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741833_1009 (size=2142) 2024-11-26T23:25:32,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741833_1009 (size=2142) 2024-11-26T23:25:32,665 DEBUG [RS:0;c707e9ace675:42261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:25:32,665 INFO [RS:0;c707e9ace675:42261 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:25:32,665 INFO [RS:0;c707e9ace675:42261 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:25:32,666 INFO [RS:0;c707e9ace675:42261 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T23:25:32,666 INFO [RS:0;c707e9ace675:42261 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:25:32,666 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:25:32,666 INFO [RS:0;c707e9ace675:42261 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42261 2024-11-26T23:25:32,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,42261,1732663516429 2024-11-26T23:25:32,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:25:32,674 INFO [RS:0;c707e9ace675:42261 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:25:32,676 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,42261,1732663516429] 2024-11-26T23:25:32,677 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,42261,1732663516429 already deleted, retry=false 2024-11-26T23:25:32,677 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of c707e9ace675,42261,1732663516429 on c707e9ace675,35437,1732663515757 2024-11-26T23:25:32,682 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure c707e9ace675,42261,1732663516429, splitWal=true, meta=true 2024-11-26T23:25:32,684 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for c707e9ace675,42261,1732663516429 (carryingMeta=true) c707e9ace675,42261,1732663516429/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@744bbb3a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T23:25:32,685 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure c707e9ace675,42261,1732663516429, splitWal=true, meta=true 2024-11-26T23:25:32,686 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure c707e9ace675,42261,1732663516429, splitWal=true, meta=true, isMeta: true 2024-11-26T23:25:32,688 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting 2024-11-26T23:25:32,689 INFO [PEWorker-1 {}] master.SplitWALManager(105): c707e9ace675,42261,1732663516429 WAL count=1, meta=true 2024-11-26T23:25:32,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta}] 2024-11-26T23:25:32,696 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=c707e9ace675,34663,1732663516528 2024-11-26T23:25:32,698 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta, worker=c707e9ace675,34663,1732663516528}] 2024-11-26T23:25:32,775 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:25:32,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:25:32,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42261-0x101569901eb0001, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:25:32,776 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1] 2024-11-26T23:25:32,777 INFO [RS:0;c707e9ace675:42261 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:25:32,777 INFO [RS:0;c707e9ace675:42261 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,42261,1732663516429; zookeeper connection closed. 2024-11-26T23:25:32,777 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11610760 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11610760 2024-11-26T23:25:32,778 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server c707e9ace675:42261 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c707e9ace675/172.17.0.2:42261 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:32,779 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1, error=java.net.ConnectException: Call to address=c707e9ace675:42261 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c707e9ace675/172.17.0.2:42261 2024-11-26T23:25:32,779 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-26T23:25:32,779 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 from cache 2024-11-26T23:25:32,779 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address c707e9ace675:42261 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c707e9ace675/172.17.0.2:42261 2024-11-26T23:25:32,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34663 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-26T23:25:32,877 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta, size=6.7 K (6845bytes) 2024-11-26T23:25:32,877 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta 2024-11-26T23:25:32,877 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta after 0ms 2024-11-26T23:25:32,880 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta took 4ms 2024-11-26T23:25:32,887 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-26T23:25:32,887 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta so closing down 2024-11-26T23:25:32,888 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:32,889 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta.temp 2024-11-26T23:25:32,890 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta.temp 2024-11-26T23:25:32,891 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:32,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741892_1070 (size=6604) 2024-11-26T23:25:32,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741892_1070 (size=6604) 2024-11-26T23:25:32,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741892_1070 (size=6604) 2024-11-26T23:25:32,901 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:32,903 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta.temp to hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-26T23:25:32,903 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 23 ms; skipped=1; WAL=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-11-26T23:25:32,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta, journal: Splitting hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta, size=6.7 K (6845bytes) at 1732663532877Finishing writing output for hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta so closing down at 1732663532887 (+10 ms)Creating recovered edits writer path=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta.temp at 1732663532890 (+3 ms)3 split writer threads finished at 1732663532891 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732663532901 (+10 ms)Rename recovered edits hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta.temp to hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732663532903 (+2 ms)Processed 16 edits across 1 Regions in 23 ms; skipped=1; WAL=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1732663532903 2024-11-26T23:25:32,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta 2024-11-26T23:25:32,904 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-26T23:25:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-26T23:25:32,911 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta to hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/oldWALs 2024-11-26T23:25:32,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-26T23:25:32,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta, worker=c707e9ace675,34663,1732663516528 in 213 msec 2024-11-26T23:25:32,915 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=c707e9ace675,34663,1732663516528 2024-11-26T23:25:32,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T23:25:32,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure c707e9ace675%2C42261%2C1732663516429.meta.1732663518314.meta, worker=c707e9ace675,34663,1732663516528 in 226 msec 2024-11-26T23:25:32,920 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting dir is empty, no logs to split. 2024-11-26T23:25:32,920 INFO [PEWorker-1 {}] master.SplitWALManager(105): c707e9ace675,42261,1732663516429 WAL count=0, meta=true 2024-11-26T23:25:32,920 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if c707e9ace675,42261,1732663516429 WAL splitting is done? wals=0, meta=true 2024-11-26T23:25:32,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T23:25:32,923 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:25:32,924 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T23:25:33,074 DEBUG [c707e9ace675:35437 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T23:25:33,074 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(204): Hosts are {c707e9ace675=0} racks are {/default-rack=0} 2024-11-26T23:25:33,075 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T23:25:33,075 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T23:25:33,075 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T23:25:33,075 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T23:25:33,075 INFO [c707e9ace675:35437 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T23:25:33,075 INFO [c707e9ace675:35437 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T23:25:33,075 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T23:25:33,075 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c707e9ace675,46791,1732663516570 2024-11-26T23:25:33,077 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,46791,1732663516570, state=OPENING 2024-11-26T23:25:33,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:25:33,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:25:33,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:25:33,079 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:25:33,079 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:33,079 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:33,079 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:33,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,46791,1732663516570}] 2024-11-26T23:25:33,085 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:25:33,086 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,46791,1732663516570, seqNum=-1] 2024-11-26T23:25:33,087 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:25:33,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:25:33,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46791 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:59626 deadline: 1732663593089, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on c707e9ace675,46791,1732663516570 2024-11-26T23:25:33,090 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c707e9ace675,46791,1732663516570, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=c707e9ace675,46791,1732663516570, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on c707e9ace675,46791,1732663516570 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:25:33,090 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=c707e9ace675,46791,1732663516570, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on c707e9ace675,46791,1732663516570 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:25:33,090 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=c707e9ace675,46791,1732663516570, seqNum=-1 from cache 2024-11-26T23:25:33,232 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:25:33,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55657, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:25:33,238 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T23:25:33,239 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:33,239 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T23:25:33,240 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C46791%2C1732663516570.meta, suffix=.meta, logDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,46791,1732663516570, archiveDir=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/oldWALs, maxLogs=32 2024-11-26T23:25:33,254 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,46791,1732663516570/c707e9ace675%2C46791%2C1732663516570.meta.1732663533241.meta, exclude list is [], retry=0 2024-11-26T23:25:33,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:33,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:33,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:33,259 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,46791,1732663516570/c707e9ace675%2C46791%2C1732663516570.meta.1732663533241.meta 2024-11-26T23:25:33,259 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:33,260 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:33,260 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T23:25:33,260 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T23:25:33,260 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T23:25:33,260 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T23:25:33,260 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:33,260 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T23:25:33,261 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T23:25:33,262 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:25:33,263 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:25:33,263 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:33,263 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:33,263 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:25:33,264 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:25:33,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:33,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:33,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:25:33,266 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:25:33,266 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:33,266 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:33,266 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:25:33,267 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:25:33,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:33,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:25:33,267 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:25:33,268 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740 2024-11-26T23:25:33,270 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740 2024-11-26T23:25:33,270 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-26T23:25:33,274 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-26T23:25:33,274 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-26T23:25:33,294 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/info/d833e7312bdb4fd9a5093c37dff94df7 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4./info:regioninfo/1732663532420/Put/seqid=0 2024-11-26T23:25:33,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741894_1072 (size=11177) 2024-11-26T23:25:33,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741894_1072 (size=11177) 2024-11-26T23:25:33,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741894_1072 (size=11177) 2024-11-26T23:25:33,301 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/info/d833e7312bdb4fd9a5093c37dff94df7 2024-11-26T23:25:33,322 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/ns/bfbc981ea22046989eeb87fd9848c227 is 43, key is default/ns:d/1732663518469/Put/seqid=0 2024-11-26T23:25:33,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741895_1073 (size=5153) 2024-11-26T23:25:33,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741895_1073 (size=5153) 2024-11-26T23:25:33,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741895_1073 (size=5153) 2024-11-26T23:25:33,330 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/ns/bfbc981ea22046989eeb87fd9848c227 2024-11-26T23:25:33,350 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/table/c148d266f5824d5aadfb561a88308894 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732663530632/Put/seqid=0 2024-11-26T23:25:33,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741896_1074 (size=5431) 2024-11-26T23:25:33,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741896_1074 (size=5431) 2024-11-26T23:25:33,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741896_1074 (size=5431) 2024-11-26T23:25:33,358 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/table/c148d266f5824d5aadfb561a88308894 2024-11-26T23:25:33,364 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/info/d833e7312bdb4fd9a5093c37dff94df7 as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/info/d833e7312bdb4fd9a5093c37dff94df7 2024-11-26T23:25:33,369 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/info/d833e7312bdb4fd9a5093c37dff94df7, entries=36, sequenceid=18, filesize=10.9 K 2024-11-26T23:25:33,370 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/ns/bfbc981ea22046989eeb87fd9848c227 as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/ns/bfbc981ea22046989eeb87fd9848c227 2024-11-26T23:25:33,376 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/ns/bfbc981ea22046989eeb87fd9848c227, entries=2, sequenceid=18, filesize=5.0 K 2024-11-26T23:25:33,377 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/table/c148d266f5824d5aadfb561a88308894 as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/table/c148d266f5824d5aadfb561a88308894 2024-11-26T23:25:33,382 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/table/c148d266f5824d5aadfb561a88308894, entries=2, sequenceid=18, filesize=5.3 K 2024-11-26T23:25:33,382 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=18, compaction requested=false; wal=null 2024-11-26T23:25:33,383 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-26T23:25:33,385 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:25:33,385 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:25:33,385 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:25:33,387 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:25:33,389 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-26T23:25:33,389 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65562053, jitterRate=-0.023049280047416687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:25:33,390 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T23:25:33,390 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732663533261Writing region info on filesystem at 1732663533261Initializing all the Stores at 1732663533261Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663533261Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663533262 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663533262Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663533262Obtaining lock to block concurrent updates at 1732663533274 (+12 ms)Preparing flush snapshotting stores in 1588230740 at 1732663533274Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732663533274Flushing stores of hbase:meta,,1.1588230740 at 1732663533274Flushing 1588230740/info: creating writer at 1732663533275 (+1 ms)Flushing 1588230740/info: appending metadata at 1732663533293 (+18 ms)Flushing 1588230740/info: closing flushed file at 1732663533293Flushing 1588230740/ns: creating writer at 1732663533307 (+14 ms)Flushing 1588230740/ns: appending metadata at 1732663533321 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732663533322 (+1 ms)Flushing 1588230740/table: creating writer at 1732663533335 (+13 ms)Flushing 1588230740/table: appending metadata at 1732663533350 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732663533350Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68c50aea: reopening flushed file at 1732663533363 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49b93786: reopening flushed file at 1732663533369 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@534e5993: reopening flushed file at 1732663533376 (+7 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=18, compaction requested=false; wal=null at 1732663533382 (+6 ms)Cleaning up temporary data from old regions at 1732663533385 (+3 ms)Running coprocessor post-open hooks at 1732663533390 (+5 ms)Region opened successfully at 1732663533390 2024-11-26T23:25:33,391 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732663533232 2024-11-26T23:25:33,393 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T23:25:33,394 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T23:25:33,394 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=c707e9ace675,46791,1732663516570 2024-11-26T23:25:33,395 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,46791,1732663516570, state=OPEN 2024-11-26T23:25:33,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:25:33,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:25:33,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:25:33,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:33,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:33,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:25:33,397 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c707e9ace675,46791,1732663516570 2024-11-26T23:25:33,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-26T23:25:33,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,46791,1732663516570 in 317 msec 2024-11-26T23:25:33,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-26T23:25:33,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 479 msec 2024-11-26T23:25:33,403 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): c707e9ace675,42261,1732663516429 had 2 regions 2024-11-26T23:25:33,404 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure c707e9ace675,42261,1732663516429, splitWal=true, meta=true, isMeta: false 2024-11-26T23:25:33,407 INFO [PEWorker-2 {}] master.SplitWALManager(105): c707e9ace675,42261,1732663516429 WAL count=1, meta=false 2024-11-26T23:25:33,407 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure c707e9ace675%2C42261%2C1732663516429.1732663517892}] 2024-11-26T23:25:33,409 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=c707e9ace675,34663,1732663516528 2024-11-26T23:25:33,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure c707e9ace675%2C42261%2C1732663516429.1732663517892, worker=c707e9ace675,34663,1732663516528}] 2024-11-26T23:25:33,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34663 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-26T23:25:33,579 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892, size=2.1 K (2142bytes) 2024-11-26T23:25:33,579 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892 2024-11-26T23:25:33,580 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892 after 1ms 2024-11-26T23:25:33,582 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892 took 3ms 2024-11-26T23:25:33,585 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for c0071604d340e73f1eaa3d5a4b74f1f4: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-26T23:25:33,585 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892 so closing down 2024-11-26T23:25:33,585 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:33,585 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:33,585 INFO [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-26T23:25:33,585 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892, journal: Splitting hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892, size=2.1 K (2142bytes) at 1732663533579Finishing writing output for hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892 so closing down at 1732663533585 (+6 ms)3 split writer threads finished at 1732663533585Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1732663533585 2024-11-26T23:25:33,585 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892 2024-11-26T23:25:33,585 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-26T23:25:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35437 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-26T23:25:33,589 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting/c707e9ace675%2C42261%2C1732663516429.1732663517892 to hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/oldWALs 2024-11-26T23:25:33,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-26T23:25:33,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c707e9ace675%2C42261%2C1732663516429.1732663517892, worker=c707e9ace675,34663,1732663516528 in 179 msec 2024-11-26T23:25:33,593 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=c707e9ace675,34663,1732663516528 2024-11-26T23:25:33,595 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:25:33,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-26T23:25:33,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure c707e9ace675%2C42261%2C1732663516429.1732663517892, worker=c707e9ace675,34663,1732663516528 in 186 msec 2024-11-26T23:25:33,596 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,46791,1732663516570, seqNum=-1] 2024-11-26T23:25:33,597 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/WALs/c707e9ace675,42261,1732663516429-splitting dir is empty, no logs to split. 2024-11-26T23:25:33,597 INFO [PEWorker-2 {}] master.SplitWALManager(105): c707e9ace675,42261,1732663516429 WAL count=0, meta=false 2024-11-26T23:25:33,597 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if c707e9ace675,42261,1732663516429 WAL splitting is done? wals=0, meta=false 2024-11-26T23:25:33,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, ASSIGN}] 2024-11-26T23:25:33,601 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, ASSIGN 2024-11-26T23:25:33,601 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T23:25:33,715 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to c707e9ace675:42261 this server is in the failed servers list 2024-11-26T23:25:33,716 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=c707e9ace675:42261 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c707e9ace675:42261 2024-11-26T23:25:33,716 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c707e9ace675:42261 2024-11-26T23:25:33,716 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=12 from cache 2024-11-26T23:25:33,752 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(204): Hosts are {c707e9ace675=0} racks are {/default-rack=0} 2024-11-26T23:25:33,752 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T23:25:33,752 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T23:25:33,752 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T23:25:33,752 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T23:25:33,752 INFO [c707e9ace675:35437 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T23:25:33,752 INFO [c707e9ace675:35437 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T23:25:33,752 DEBUG [c707e9ace675:35437 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T23:25:33,753 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=OPENING, regionLocation=c707e9ace675,46791,1732663516570 2024-11-26T23:25:33,754 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server c707e9ace675:42261 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c707e9ace675/172.17.0.2:42261 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:33,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1, error=java.net.ConnectException: Call to address=c707e9ace675:42261 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c707e9ace675/172.17.0.2:42261 2024-11-26T23:25:33,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-26T23:25:33,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=c707e9ace675,42261,1732663516429, seqNum=-1 from cache 2024-11-26T23:25:33,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address c707e9ace675:42261 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: c707e9ace675/172.17.0.2:42261 2024-11-26T23:25:33,865 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:25:33,866 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,46791,1732663516570, seqNum=-1] 2024-11-26T23:25:33,866 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:25:33,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41961, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:25:33,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, ASSIGN because future has completed 2024-11-26T23:25:33,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,46791,1732663516570}] 2024-11-26T23:25:33,931 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=18] 2024-11-26T23:25:33,931 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.AbstractRpcClient(357): Not trying to connect to c707e9ace675:42261 this server is in the failed servers list 2024-11-26T23:25:33,932 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=c707e9ace675:42261 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c707e9ace675:42261 2024-11-26T23:25:33,932 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c707e9ace675:42261 2024-11-26T23:25:33,932 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,42261,1732663516429, seqNum=18 from cache 2024-11-26T23:25:34,027 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:34,028 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => c0071604d340e73f1eaa3d5a4b74f1f4, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:34,028 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,028 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:34,028 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,029 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,030 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,031 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf1 2024-11-26T23:25:34,031 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,038 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf1/a8cdce13a493427c808d8c5c9f337b25 2024-11-26T23:25:34,038 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,038 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,039 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0071604d340e73f1eaa3d5a4b74f1f4 columnFamilyName cf2 2024-11-26T23:25:34,039 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,045 DEBUG [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/cf2/6a1b00d3e18c488cb0588ccea9fc9a42 2024-11-26T23:25:34,046 INFO [StoreOpener-c0071604d340e73f1eaa3d5a4b74f1f4-1 {}] regionserver.HStore(327): Store=c0071604d340e73f1eaa3d5a4b74f1f4/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,046 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,047 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,048 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,049 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,049 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,049 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-26T23:25:34,051 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,052 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened c0071604d340e73f1eaa3d5a4b74f1f4; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70167189, jitterRate=0.04557259380817413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-26T23:25:34,052 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:34,052 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for c0071604d340e73f1eaa3d5a4b74f1f4: Running coprocessor pre-open hook at 1732663534029Writing region info on filesystem at 1732663534029Initializing all the Stores at 1732663534030 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534030Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534030Cleaning up temporary data from old regions at 1732663534049 (+19 ms)Running coprocessor post-open hooks at 1732663534052 (+3 ms)Region opened successfully at 1732663534052 2024-11-26T23:25:34,054 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., pid=21, masterSystemTime=1732663534023 2024-11-26T23:25:34,056 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:34,056 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:34,057 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=c0071604d340e73f1eaa3d5a4b74f1f4, regionState=OPEN, openSeqNum=18, regionLocation=c707e9ace675,46791,1732663516570 2024-11-26T23:25:34,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,46791,1732663516570 because future has completed 2024-11-26T23:25:34,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-26T23:25:34,063 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure c0071604d340e73f1eaa3d5a4b74f1f4, server=c707e9ace675,46791,1732663516570 in 190 msec 2024-11-26T23:25:34,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-26T23:25:34,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=c0071604d340e73f1eaa3d5a4b74f1f4, ASSIGN in 464 msec 2024-11-26T23:25:34,065 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server c707e9ace675,42261,1732663516429 after splitting done 2024-11-26T23:25:34,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure c707e9ace675,42261,1732663516429, splitWal=true, meta=true in 1.3880 sec 2024-11-26T23:25:34,240 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4., hostname=c707e9ace675,46791,1732663516570, seqNum=18] 2024-11-26T23:25:34,261 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 412), OpenFileDescriptor=1052 (was 1024) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=318 (was 318), ProcessCount=11 (was 11), AvailableMemoryMB=6856 (was 6925) 2024-11-26T23:25:34,264 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1052 is superior to 1024 2024-11-26T23:25:34,280 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=410, OpenFileDescriptor=1052, MaxFileDescriptor=1048576, SystemLoadAverage=318, ProcessCount=11, AvailableMemoryMB=6856 2024-11-26T23:25:34,280 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1052 is superior to 1024 2024-11-26T23:25:34,297 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:34,299 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:34,300 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:34,302 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-12835927, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/hregion-12835927, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:34,315 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-12835927/hregion-12835927.1732663534303, exclude list is [], retry=0 2024-11-26T23:25:34,318 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:34,318 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:34,318 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:34,321 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-12835927/hregion-12835927.1732663534303 2024-11-26T23:25:34,321 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:34,321 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 437f68ca0a81d9b2065233326bf950c0, NAME => 'testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/hbase 2024-11-26T23:25:34,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741898_1076 (size=67) 2024-11-26T23:25:34,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741898_1076 (size=67) 2024-11-26T23:25:34,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741898_1076 (size=67) 2024-11-26T23:25:34,331 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:34,332 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,334 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437f68ca0a81d9b2065233326bf950c0 columnFamilyName a 2024-11-26T23:25:34,334 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,334 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(327): Store=437f68ca0a81d9b2065233326bf950c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,335 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,336 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437f68ca0a81d9b2065233326bf950c0 columnFamilyName b 2024-11-26T23:25:34,336 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,336 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(327): Store=437f68ca0a81d9b2065233326bf950c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,336 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,337 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437f68ca0a81d9b2065233326bf950c0 columnFamilyName c 2024-11-26T23:25:34,337 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,338 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(327): Store=437f68ca0a81d9b2065233326bf950c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,338 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,339 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,339 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,340 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,340 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,340 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:34,341 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,343 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:34,344 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 437f68ca0a81d9b2065233326bf950c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65463426, jitterRate=-0.0245189368724823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:34,345 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 437f68ca0a81d9b2065233326bf950c0: Writing region info on filesystem at 1732663534331Initializing all the Stores at 1732663534332 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534332Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534332Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534332Cleaning up temporary data from old regions at 1732663534340 (+8 ms)Region opened successfully at 1732663534344 (+4 ms) 2024-11-26T23:25:34,345 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 437f68ca0a81d9b2065233326bf950c0, disabling compactions & flushes 2024-11-26T23:25:34,345 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. 2024-11-26T23:25:34,345 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. 2024-11-26T23:25:34,345 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. after waiting 0 ms 2024-11-26T23:25:34,345 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. 2024-11-26T23:25:34,345 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. 2024-11-26T23:25:34,345 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 437f68ca0a81d9b2065233326bf950c0: Waiting for close lock at 1732663534345Disabling compacts and flushes for region at 1732663534345Disabling writes for close at 1732663534345Writing region close event to WAL at 1732663534345Closed at 1732663534345 2024-11-26T23:25:34,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741897_1075 (size=93) 2024-11-26T23:25:34,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741897_1075 (size=93) 2024-11-26T23:25:34,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741897_1075 (size=93) 2024-11-26T23:25:34,352 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:34,352 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-12835927:(num 1732663534303) 2024-11-26T23:25:34,352 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:34,354 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:34,367 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354, exclude list is [], retry=0 2024-11-26T23:25:34,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:34,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:34,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:34,372 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354 2024-11-26T23:25:34,372 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:34,372 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 437f68ca0a81d9b2065233326bf950c0, NAME => 'testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:34,372 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:34,372 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,372 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,374 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,375 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437f68ca0a81d9b2065233326bf950c0 columnFamilyName a 2024-11-26T23:25:34,375 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,375 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(327): Store=437f68ca0a81d9b2065233326bf950c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,375 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,376 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437f68ca0a81d9b2065233326bf950c0 columnFamilyName b 2024-11-26T23:25:34,376 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,376 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(327): Store=437f68ca0a81d9b2065233326bf950c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,377 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,377 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437f68ca0a81d9b2065233326bf950c0 columnFamilyName c 2024-11-26T23:25:34,377 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,378 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(327): Store=437f68ca0a81d9b2065233326bf950c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,378 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,378 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,379 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,380 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,380 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,381 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:34,382 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,383 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 437f68ca0a81d9b2065233326bf950c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64859379, jitterRate=-0.033519938588142395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:34,384 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 437f68ca0a81d9b2065233326bf950c0: Writing region info on filesystem at 1732663534373Initializing all the Stores at 1732663534373Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534373Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534374 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534374Cleaning up temporary data from old regions at 1732663534380 (+6 ms)Region opened successfully at 1732663534384 (+4 ms) 2024-11-26T23:25:34,413 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 437f68ca0a81d9b2065233326bf950c0 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-26T23:25:34,432 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/a/ddf2eb24adfc4784bfd5bcdf29119dd7 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732663534384/Put/seqid=0 2024-11-26T23:25:34,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741900_1078 (size=5958) 2024-11-26T23:25:34,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741900_1078 (size=5958) 2024-11-26T23:25:34,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741900_1078 (size=5958) 2024-11-26T23:25:34,440 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/a/ddf2eb24adfc4784bfd5bcdf29119dd7 2024-11-26T23:25:34,459 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/b/e5832d1b9a514b30becdea918a5bed75 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732663534395/Put/seqid=0 2024-11-26T23:25:34,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741901_1079 (size=5958) 2024-11-26T23:25:34,466 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/b/e5832d1b9a514b30becdea918a5bed75 2024-11-26T23:25:34,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741901_1079 (size=5958) 2024-11-26T23:25:34,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741901_1079 (size=5958) 2024-11-26T23:25:34,493 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/c/04dadb8ed9354d8a87bd69e6d5c60565 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732663534403/Put/seqid=0 2024-11-26T23:25:34,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741902_1080 (size=5958) 2024-11-26T23:25:34,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741902_1080 (size=5958) 2024-11-26T23:25:34,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741902_1080 (size=5958) 2024-11-26T23:25:34,501 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/c/04dadb8ed9354d8a87bd69e6d5c60565 2024-11-26T23:25:34,508 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/a/ddf2eb24adfc4784bfd5bcdf29119dd7 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/a/ddf2eb24adfc4784bfd5bcdf29119dd7 2024-11-26T23:25:34,514 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/a/ddf2eb24adfc4784bfd5bcdf29119dd7, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T23:25:34,515 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/b/e5832d1b9a514b30becdea918a5bed75 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/b/e5832d1b9a514b30becdea918a5bed75 2024-11-26T23:25:34,522 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/b/e5832d1b9a514b30becdea918a5bed75, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T23:25:34,523 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/c/04dadb8ed9354d8a87bd69e6d5c60565 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/c/04dadb8ed9354d8a87bd69e6d5c60565 2024-11-26T23:25:34,529 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/c/04dadb8ed9354d8a87bd69e6d5c60565, entries=10, sequenceid=33, filesize=5.8 K 2024-11-26T23:25:34,531 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 437f68ca0a81d9b2065233326bf950c0 in 118ms, sequenceid=33, compaction requested=false 2024-11-26T23:25:34,531 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 437f68ca0a81d9b2065233326bf950c0: 2024-11-26T23:25:34,531 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 437f68ca0a81d9b2065233326bf950c0, disabling compactions & flushes 2024-11-26T23:25:34,531 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. 2024-11-26T23:25:34,531 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. 2024-11-26T23:25:34,531 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. after waiting 0 ms 2024-11-26T23:25:34,531 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. 2024-11-26T23:25:34,533 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. 2024-11-26T23:25:34,533 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 437f68ca0a81d9b2065233326bf950c0: Waiting for close lock at 1732663534531Disabling compacts and flushes for region at 1732663534531Disabling writes for close at 1732663534531Writing region close event to WAL at 1732663534533 (+2 ms)Closed at 1732663534533 2024-11-26T23:25:34,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741899_1077 (size=5937) 2024-11-26T23:25:34,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741899_1077 (size=5937) 2024-11-26T23:25:34,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741899_1077 (size=5937) 2024-11-26T23:25:34,541 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/b/e5832d1b9a514b30becdea918a5bed75 to hdfs://localhost:45925/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/b/e5832d1b9a514b30becdea918a5bed75 2024-11-26T23:25:34,565 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354, size=5.8 K (5937bytes) 2024-11-26T23:25:34,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354 2024-11-26T23:25:34,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354 after 0ms 2024-11-26T23:25:34,569 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354 took 4ms 2024-11-26T23:25:34,571 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354 so closing down 2024-11-26T23:25:34,571 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:34,572 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732663534354.temp 2024-11-26T23:25:34,573 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000003-wal.1732663534354.temp 2024-11-26T23:25:34,573 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:34,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741903_1081 (size=5223) 2024-11-26T23:25:34,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741903_1081 (size=5223) 2024-11-26T23:25:34,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741903_1081 (size=5223) 2024-11-26T23:25:34,583 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000003-wal.1732663534354.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:34,584 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000003-wal.1732663534354.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000032 2024-11-26T23:25:34,584 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-26T23:25:34,585 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354, journal: Splitting hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354, size=5.8 K (5937bytes) at 1732663534565Finishing writing output for hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354 so closing down at 1732663534571 (+6 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000003-wal.1732663534354.temp at 1732663534573 (+2 ms)3 split writer threads finished at 1732663534573Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000003-wal.1732663534354.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732663534583 (+10 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000003-wal.1732663534354.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000032 at 1732663534584 (+1 ms)Processed 32 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1732663534584 2024-11-26T23:25:34,586 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534354 to hdfs://localhost:45925/hbase/oldWALs/wal.1732663534354 2024-11-26T23:25:34,587 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000032 2024-11-26T23:25:34,588 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:34,589 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:34,610 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534590, exclude list is [], retry=0 2024-11-26T23:25:34,613 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:34,614 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:34,614 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:34,616 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534590 2024-11-26T23:25:34,619 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:34,619 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 437f68ca0a81d9b2065233326bf950c0, NAME => 'testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:34,619 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:34,619 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,619 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,621 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,622 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437f68ca0a81d9b2065233326bf950c0 columnFamilyName a 2024-11-26T23:25:34,622 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,628 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/a/ddf2eb24adfc4784bfd5bcdf29119dd7 2024-11-26T23:25:34,628 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(327): Store=437f68ca0a81d9b2065233326bf950c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,629 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,630 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437f68ca0a81d9b2065233326bf950c0 columnFamilyName b 2024-11-26T23:25:34,630 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,631 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(327): Store=437f68ca0a81d9b2065233326bf950c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,631 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,632 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437f68ca0a81d9b2065233326bf950c0 columnFamilyName c 2024-11-26T23:25:34,632 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,637 DEBUG [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/c/04dadb8ed9354d8a87bd69e6d5c60565 2024-11-26T23:25:34,638 INFO [StoreOpener-437f68ca0a81d9b2065233326bf950c0-1 {}] regionserver.HStore(327): Store=437f68ca0a81d9b2065233326bf950c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,638 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,639 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,640 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,641 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000032 2024-11-26T23:25:34,644 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000032 2024-11-26T23:25:34,644 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 437f68ca0a81d9b2065233326bf950c0 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-26T23:25:34,660 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/b/d1e5b41ab1104406b6b244628933c202 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732663534395/Put/seqid=0 2024-11-26T23:25:34,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741905_1083 (size=5958) 2024-11-26T23:25:34,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741905_1083 (size=5958) 2024-11-26T23:25:34,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741905_1083 (size=5958) 2024-11-26T23:25:34,669 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/b/d1e5b41ab1104406b6b244628933c202 2024-11-26T23:25:34,675 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/.tmp/b/d1e5b41ab1104406b6b244628933c202 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/b/d1e5b41ab1104406b6b244628933c202 2024-11-26T23:25:34,680 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/b/d1e5b41ab1104406b6b244628933c202, entries=10, sequenceid=32, filesize=5.8 K 2024-11-26T23:25:34,680 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 437f68ca0a81d9b2065233326bf950c0 in 36ms, sequenceid=32, compaction requested=false; wal=null 2024-11-26T23:25:34,681 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/0000000000000000032 2024-11-26T23:25:34,682 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,682 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,683 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:34,684 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 437f68ca0a81d9b2065233326bf950c0 2024-11-26T23:25:34,686 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/437f68ca0a81d9b2065233326bf950c0/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-26T23:25:34,687 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 437f68ca0a81d9b2065233326bf950c0; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65247248, jitterRate=-0.0277402400970459}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:34,688 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 437f68ca0a81d9b2065233326bf950c0: Writing region info on filesystem at 1732663534619Initializing all the Stores at 1732663534620 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534620Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534621 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534621Obtaining lock to block concurrent updates at 1732663534644 (+23 ms)Preparing flush snapshotting stores in 437f68ca0a81d9b2065233326bf950c0 at 1732663534644Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732663534644Flushing stores of testReplayEditsWrittenViaHRegion,,1732663534298.437f68ca0a81d9b2065233326bf950c0. at 1732663534644Flushing 437f68ca0a81d9b2065233326bf950c0/b: creating writer at 1732663534645 (+1 ms)Flushing 437f68ca0a81d9b2065233326bf950c0/b: appending metadata at 1732663534660 (+15 ms)Flushing 437f68ca0a81d9b2065233326bf950c0/b: closing flushed file at 1732663534660Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4da90806: reopening flushed file at 1732663534674 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 437f68ca0a81d9b2065233326bf950c0 in 36ms, sequenceid=32, compaction requested=false; wal=null at 1732663534680 (+6 ms)Cleaning up temporary data from old regions at 1732663534682 (+2 ms)Region opened successfully at 1732663534688 (+6 ms) 2024-11-26T23:25:34,713 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=419 (was 410) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:50670 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:56490 [Waiting for operation #57] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:37138 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:39632 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:57718 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:53988 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1126 (was 1052) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=316 (was 318), ProcessCount=11 (was 11), AvailableMemoryMB=6849 (was 6856) 2024-11-26T23:25:34,714 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-11-26T23:25:34,730 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417, OpenFileDescriptor=1126, MaxFileDescriptor=1048576, SystemLoadAverage=316, ProcessCount=11, AvailableMemoryMB=6849 2024-11-26T23:25:34,730 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-11-26T23:25:34,746 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:34,748 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:34,749 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:34,751 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45064701, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/hregion-45064701, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:34,765 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45064701/hregion-45064701.1732663534752, exclude list is [], retry=0 2024-11-26T23:25:34,767 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:34,768 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:34,768 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:34,770 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45064701/hregion-45064701.1732663534752 2024-11-26T23:25:34,771 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:34,771 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ba425b95dc2a0fadc5dad4fff9fed1af, NAME => 'testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/hbase 2024-11-26T23:25:34,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741907_1085 (size=68) 2024-11-26T23:25:34,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741907_1085 (size=68) 2024-11-26T23:25:34,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741907_1085 (size=68) 2024-11-26T23:25:34,783 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:34,785 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,787 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba425b95dc2a0fadc5dad4fff9fed1af columnFamilyName a 2024-11-26T23:25:34,787 DEBUG [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,788 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(327): Store=ba425b95dc2a0fadc5dad4fff9fed1af/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,788 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,789 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba425b95dc2a0fadc5dad4fff9fed1af columnFamilyName b 2024-11-26T23:25:34,789 DEBUG [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,790 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(327): Store=ba425b95dc2a0fadc5dad4fff9fed1af/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,790 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,791 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba425b95dc2a0fadc5dad4fff9fed1af columnFamilyName c 2024-11-26T23:25:34,791 DEBUG [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,792 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(327): Store=ba425b95dc2a0fadc5dad4fff9fed1af/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,792 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,793 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,793 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,794 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,794 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,795 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:34,796 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,798 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:34,798 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ba425b95dc2a0fadc5dad4fff9fed1af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66397121, jitterRate=-0.010605797171592712}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:34,799 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ba425b95dc2a0fadc5dad4fff9fed1af: Writing region info on filesystem at 1732663534784Initializing all the Stores at 1732663534784Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534785 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534785Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534785Cleaning up temporary data from old regions at 1732663534794 (+9 ms)Region opened successfully at 1732663534799 (+5 ms) 2024-11-26T23:25:34,799 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ba425b95dc2a0fadc5dad4fff9fed1af, disabling compactions & flushes 2024-11-26T23:25:34,799 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. 2024-11-26T23:25:34,799 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. 2024-11-26T23:25:34,799 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. after waiting 0 ms 2024-11-26T23:25:34,799 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. 2024-11-26T23:25:34,800 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. 2024-11-26T23:25:34,800 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ba425b95dc2a0fadc5dad4fff9fed1af: Waiting for close lock at 1732663534799Disabling compacts and flushes for region at 1732663534799Disabling writes for close at 1732663534799Writing region close event to WAL at 1732663534800 (+1 ms)Closed at 1732663534800 2024-11-26T23:25:34,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741906_1084 (size=93) 2024-11-26T23:25:34,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741906_1084 (size=93) 2024-11-26T23:25:34,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741906_1084 (size=93) 2024-11-26T23:25:34,806 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:34,806 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45064701:(num 1732663534752) 2024-11-26T23:25:34,806 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:34,807 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:34,821 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807, exclude list is [], retry=0 2024-11-26T23:25:34,824 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:34,825 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:34,825 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:34,827 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807 2024-11-26T23:25:34,827 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:34,880 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ba425b95dc2a0fadc5dad4fff9fed1af, NAME => 'testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:34,882 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,882 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:34,882 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,883 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,885 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,886 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba425b95dc2a0fadc5dad4fff9fed1af columnFamilyName a 2024-11-26T23:25:34,886 DEBUG [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,886 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(327): Store=ba425b95dc2a0fadc5dad4fff9fed1af/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,886 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,887 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba425b95dc2a0fadc5dad4fff9fed1af columnFamilyName b 2024-11-26T23:25:34,887 DEBUG [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,888 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(327): Store=ba425b95dc2a0fadc5dad4fff9fed1af/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,888 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,888 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba425b95dc2a0fadc5dad4fff9fed1af columnFamilyName c 2024-11-26T23:25:34,888 DEBUG [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:34,889 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(327): Store=ba425b95dc2a0fadc5dad4fff9fed1af/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:34,889 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,890 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,891 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,891 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,892 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,892 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:34,893 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,894 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ba425b95dc2a0fadc5dad4fff9fed1af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61359639, jitterRate=-0.08567012846469879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:34,894 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:34,895 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ba425b95dc2a0fadc5dad4fff9fed1af: Running coprocessor pre-open hook at 1732663534883Writing region info on filesystem at 1732663534883Initializing all the Stores at 1732663534884 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534884Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534884Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663534884Cleaning up temporary data from old regions at 1732663534892 (+8 ms)Running coprocessor post-open hooks at 1732663534894 (+2 ms)Region opened successfully at 1732663534895 (+1 ms) 2024-11-26T23:25:34,909 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ba425b95dc2a0fadc5dad4fff9fed1af 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-26T23:25:34,910 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ba425b95dc2a0fadc5dad4fff9fed1af/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:35,911 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ba425b95dc2a0fadc5dad4fff9fed1af/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:36,130 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:25:36,161 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T23:25:36,167 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-26T23:25:36,167 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:36,169 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-26T23:25:36,169 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:36,169 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-26T23:25:36,169 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:36,911 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ba425b95dc2a0fadc5dad4fff9fed1af/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:37,912 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ba425b95dc2a0fadc5dad4fff9fed1af/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:38,913 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ba425b95dc2a0fadc5dad4fff9fed1af/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:39,913 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ba425b95dc2a0fadc5dad4fff9fed1af/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:40,914 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ba425b95dc2a0fadc5dad4fff9fed1af/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:41,915 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ba425b95dc2a0fadc5dad4fff9fed1af/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:42,916 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ba425b95dc2a0fadc5dad4fff9fed1af/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:43,916 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ba425b95dc2a0fadc5dad4fff9fed1af/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:43,917 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ba425b95dc2a0fadc5dad4fff9fed1af: 2024-11-26T23:25:43,917 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. 2024-11-26T23:25:43,930 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ba425b95dc2a0fadc5dad4fff9fed1af: 2024-11-26T23:25:43,930 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-26T23:25:43,930 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ba425b95dc2a0fadc5dad4fff9fed1af, disabling compactions & flushes 2024-11-26T23:25:43,930 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. 2024-11-26T23:25:43,930 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. 2024-11-26T23:25:43,930 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. after waiting 0 ms 2024-11-26T23:25:43,930 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. 2024-11-26T23:25:43,930 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. 2024-11-26T23:25:43,931 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. 2024-11-26T23:25:43,931 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ba425b95dc2a0fadc5dad4fff9fed1af: Waiting for close lock at 1732663543930Running coprocessor pre-close hooks at 1732663543930Disabling compacts and flushes for region at 1732663543930Disabling writes for close at 1732663543930Writing region close event to WAL at 1732663543930Running coprocessor post-close hooks at 1732663543931 (+1 ms)Closed at 1732663543931 2024-11-26T23:25:43,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741908_1086 (size=3941) 2024-11-26T23:25:43,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741908_1086 (size=3941) 2024-11-26T23:25:43,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741908_1086 (size=3941) 2024-11-26T23:25:43,954 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807, size=3.8 K (3941bytes) 2024-11-26T23:25:43,955 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807 2024-11-26T23:25:43,955 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807 after 0ms 2024-11-26T23:25:43,957 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807 took 3ms 2024-11-26T23:25:43,959 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807 so closing down 2024-11-26T23:25:43,959 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:43,960 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732663534807.temp 2024-11-26T23:25:43,961 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000004-wal.1732663534807.temp 2024-11-26T23:25:43,962 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:43,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741909_1087 (size=2983) 2024-11-26T23:25:43,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741909_1087 (size=2983) 2024-11-26T23:25:43,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741909_1087 (size=2983) 2024-11-26T23:25:43,969 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000004-wal.1732663534807.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:43,970 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000004-wal.1732663534807.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000026 2024-11-26T23:25:43,971 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-26T23:25:43,971 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807, journal: Splitting hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807, size=3.8 K (3941bytes) at 1732663543955Finishing writing output for hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807 so closing down at 1732663543959 (+4 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000004-wal.1732663534807.temp at 1732663543961 (+2 ms)3 split writer threads finished at 1732663543962 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000004-wal.1732663534807.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732663543969 (+7 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000004-wal.1732663534807.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000026 at 1732663543971 (+2 ms)Processed 23 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1732663543971 2024-11-26T23:25:43,972 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663534807 to hdfs://localhost:45925/hbase/oldWALs/wal.1732663534807 2024-11-26T23:25:43,973 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000026 2024-11-26T23:25:43,973 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:43,975 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:43,988 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663543975, exclude list is [], retry=0 2024-11-26T23:25:43,990 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:43,991 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:43,991 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:43,992 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663543975 2024-11-26T23:25:43,993 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:43,993 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ba425b95dc2a0fadc5dad4fff9fed1af, NAME => 'testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:43,993 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:43,993 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:43,994 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:43,994 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:43,995 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:43,996 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba425b95dc2a0fadc5dad4fff9fed1af columnFamilyName a 2024-11-26T23:25:43,996 DEBUG [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:43,996 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(327): Store=ba425b95dc2a0fadc5dad4fff9fed1af/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:43,997 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:43,997 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba425b95dc2a0fadc5dad4fff9fed1af columnFamilyName b 2024-11-26T23:25:43,997 DEBUG [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:43,998 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(327): Store=ba425b95dc2a0fadc5dad4fff9fed1af/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:43,998 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:43,999 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba425b95dc2a0fadc5dad4fff9fed1af columnFamilyName c 2024-11-26T23:25:43,999 DEBUG [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:43,999 INFO [StoreOpener-ba425b95dc2a0fadc5dad4fff9fed1af-1 {}] regionserver.HStore(327): Store=ba425b95dc2a0fadc5dad4fff9fed1af/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:44,000 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:44,000 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:44,002 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:44,003 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000026 2024-11-26T23:25:44,007 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000026 2024-11-26T23:25:44,007 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ba425b95dc2a0fadc5dad4fff9fed1af 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-26T23:25:44,022 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/.tmp/a/83bfb0032d3b4b88a8b8b80baeb31d14 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732663543921/Put/seqid=0 2024-11-26T23:25:44,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741911_1089 (size=5523) 2024-11-26T23:25:44,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741911_1089 (size=5523) 2024-11-26T23:25:44,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741911_1089 (size=5523) 2024-11-26T23:25:44,434 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/.tmp/a/83bfb0032d3b4b88a8b8b80baeb31d14 2024-11-26T23:25:44,455 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/.tmp/b/f3af6764c1c246c193f41f2e8c150ad7 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732663543918/Put/seqid=0 2024-11-26T23:25:44,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741912_1090 (size=5524) 2024-11-26T23:25:44,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741912_1090 (size=5524) 2024-11-26T23:25:44,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741912_1090 (size=5524) 2024-11-26T23:25:44,462 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/.tmp/b/f3af6764c1c246c193f41f2e8c150ad7 2024-11-26T23:25:44,482 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/.tmp/c/e7369f908ea74f1390306c0b64729a59 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732663543919/Put/seqid=0 2024-11-26T23:25:44,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741913_1091 (size=5457) 2024-11-26T23:25:44,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741913_1091 (size=5457) 2024-11-26T23:25:44,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741913_1091 (size=5457) 2024-11-26T23:25:44,490 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/.tmp/c/e7369f908ea74f1390306c0b64729a59 2024-11-26T23:25:44,495 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/.tmp/a/83bfb0032d3b4b88a8b8b80baeb31d14 as hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/a/83bfb0032d3b4b88a8b8b80baeb31d14 2024-11-26T23:25:44,500 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/a/83bfb0032d3b4b88a8b8b80baeb31d14, entries=7, sequenceid=26, filesize=5.4 K 2024-11-26T23:25:44,501 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/.tmp/b/f3af6764c1c246c193f41f2e8c150ad7 as hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/b/f3af6764c1c246c193f41f2e8c150ad7 2024-11-26T23:25:44,506 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/b/f3af6764c1c246c193f41f2e8c150ad7, entries=7, sequenceid=26, filesize=5.4 K 2024-11-26T23:25:44,507 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/.tmp/c/e7369f908ea74f1390306c0b64729a59 as hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/c/e7369f908ea74f1390306c0b64729a59 2024-11-26T23:25:44,511 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/c/e7369f908ea74f1390306c0b64729a59, entries=6, sequenceid=26, filesize=5.3 K 2024-11-26T23:25:44,512 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for ba425b95dc2a0fadc5dad4fff9fed1af in 505ms, sequenceid=26, compaction requested=false; wal=null 2024-11-26T23:25:44,512 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/0000000000000000026 2024-11-26T23:25:44,513 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:44,513 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:44,514 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:44,515 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:44,518 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsAfterAbortingFlush/ba425b95dc2a0fadc5dad4fff9fed1af/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-26T23:25:44,519 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ba425b95dc2a0fadc5dad4fff9fed1af; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67916110, jitterRate=0.012028902769088745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:44,519 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba425b95dc2a0fadc5dad4fff9fed1af 2024-11-26T23:25:44,519 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ba425b95dc2a0fadc5dad4fff9fed1af: Running coprocessor pre-open hook at 1732663543994Writing region info on filesystem at 1732663543994Initializing all the Stores at 1732663543994Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663543994Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663543995 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663543995Obtaining lock to block concurrent updates at 1732663544007 (+12 ms)Preparing flush snapshotting stores in ba425b95dc2a0fadc5dad4fff9fed1af at 1732663544007Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732663544007Flushing stores of testReplayEditsAfterAbortingFlush,,1732663534747.ba425b95dc2a0fadc5dad4fff9fed1af. at 1732663544007Flushing ba425b95dc2a0fadc5dad4fff9fed1af/a: creating writer at 1732663544007Flushing ba425b95dc2a0fadc5dad4fff9fed1af/a: appending metadata at 1732663544022 (+15 ms)Flushing ba425b95dc2a0fadc5dad4fff9fed1af/a: closing flushed file at 1732663544022Flushing ba425b95dc2a0fadc5dad4fff9fed1af/b: creating writer at 1732663544440 (+418 ms)Flushing ba425b95dc2a0fadc5dad4fff9fed1af/b: appending metadata at 1732663544455 (+15 ms)Flushing ba425b95dc2a0fadc5dad4fff9fed1af/b: closing flushed file at 1732663544455Flushing ba425b95dc2a0fadc5dad4fff9fed1af/c: creating writer at 1732663544468 (+13 ms)Flushing ba425b95dc2a0fadc5dad4fff9fed1af/c: appending metadata at 1732663544482 (+14 ms)Flushing ba425b95dc2a0fadc5dad4fff9fed1af/c: closing flushed file at 1732663544482Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c6a4e50: reopening flushed file at 1732663544495 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57dd555a: reopening flushed file at 1732663544501 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71598366: reopening flushed file at 1732663544506 (+5 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for ba425b95dc2a0fadc5dad4fff9fed1af in 505ms, sequenceid=26, compaction requested=false; wal=null at 1732663544512 (+6 ms)Cleaning up temporary data from old regions at 1732663544513 (+1 ms)Running coprocessor post-open hooks at 1732663544519 (+6 ms)Region opened successfully at 1732663544519 2024-11-26T23:25:44,540 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 417), OpenFileDescriptor=1182 (was 1126) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=276 (was 316), ProcessCount=11 (was 11), AvailableMemoryMB=6831 (was 6849) 2024-11-26T23:25:44,540 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-26T23:25:44,553 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=276, ProcessCount=11, AvailableMemoryMB=6831 2024-11-26T23:25:44,553 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-26T23:25:44,568 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:44,570 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:44,571 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:44,573 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-86778206, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/hregion-86778206, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:44,586 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-86778206/hregion-86778206.1732663544573, exclude list is [], retry=0 2024-11-26T23:25:44,588 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:44,588 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:44,589 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:44,590 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-86778206/hregion-86778206.1732663544573 2024-11-26T23:25:44,591 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:44,591 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8acd1607604cbca6312ea4449ebda85e, NAME => 'testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/hbase 2024-11-26T23:25:44,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741915_1093 (size=61) 2024-11-26T23:25:44,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741915_1093 (size=61) 2024-11-26T23:25:44,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741915_1093 (size=61) 2024-11-26T23:25:44,601 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:44,603 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,604 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8acd1607604cbca6312ea4449ebda85e columnFamilyName a 2024-11-26T23:25:44,604 DEBUG [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:44,604 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] regionserver.HStore(327): Store=8acd1607604cbca6312ea4449ebda85e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:44,605 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,605 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,605 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,606 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,606 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,607 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,609 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:44,610 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8acd1607604cbca6312ea4449ebda85e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65216776, jitterRate=-0.028194308280944824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:25:44,610 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8acd1607604cbca6312ea4449ebda85e: Writing region info on filesystem at 1732663544601Initializing all the Stores at 1732663544602 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663544602Cleaning up temporary data from old regions at 1732663544606 (+4 ms)Region opened successfully at 1732663544610 (+4 ms) 2024-11-26T23:25:44,610 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8acd1607604cbca6312ea4449ebda85e, disabling compactions & flushes 2024-11-26T23:25:44,610 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. 2024-11-26T23:25:44,610 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. 2024-11-26T23:25:44,610 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. after waiting 0 ms 2024-11-26T23:25:44,610 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. 2024-11-26T23:25:44,611 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. 2024-11-26T23:25:44,611 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8acd1607604cbca6312ea4449ebda85e: Waiting for close lock at 1732663544610Disabling compacts and flushes for region at 1732663544610Disabling writes for close at 1732663544610Writing region close event to WAL at 1732663544611 (+1 ms)Closed at 1732663544611 2024-11-26T23:25:44,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741914_1092 (size=93) 2024-11-26T23:25:44,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741914_1092 (size=93) 2024-11-26T23:25:44,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741914_1092 (size=93) 2024-11-26T23:25:44,617 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:44,617 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-86778206:(num 1732663544573) 2024-11-26T23:25:44,617 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:44,619 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:44,638 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620, exclude list is [], retry=0 2024-11-26T23:25:44,641 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:44,641 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:44,642 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:44,643 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620 2024-11-26T23:25:44,644 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:44,644 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8acd1607604cbca6312ea4449ebda85e, NAME => 'testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:44,644 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:44,644 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,644 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,646 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,647 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8acd1607604cbca6312ea4449ebda85e columnFamilyName a 2024-11-26T23:25:44,647 DEBUG [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:44,648 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] regionserver.HStore(327): Store=8acd1607604cbca6312ea4449ebda85e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:44,648 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,648 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,649 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,650 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,650 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,651 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:44,652 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8acd1607604cbca6312ea4449ebda85e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62103992, jitterRate=-0.07457840442657471}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:25:44,653 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8acd1607604cbca6312ea4449ebda85e: Writing region info on filesystem at 1732663544644Initializing all the Stores at 1732663544645 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663544645Cleaning up temporary data from old regions at 1732663544650 (+5 ms)Region opened successfully at 1732663544653 (+3 ms) 2024-11-26T23:25:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8acd1607604cbca6312ea4449ebda85e, disabling compactions & flushes 2024-11-26T23:25:44,662 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. 2024-11-26T23:25:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. 2024-11-26T23:25:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. after waiting 0 ms 2024-11-26T23:25:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. 2024-11-26T23:25:44,662 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. 2024-11-26T23:25:44,662 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. 2024-11-26T23:25:44,662 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8acd1607604cbca6312ea4449ebda85e: Waiting for close lock at 1732663544662Disabling compacts and flushes for region at 1732663544662Disabling writes for close at 1732663544662Writing region close event to WAL at 1732663544662Closed at 1732663544662 2024-11-26T23:25:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741916_1094 (size=1623) 2024-11-26T23:25:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741916_1094 (size=1623) 2024-11-26T23:25:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741916_1094 (size=1623) 2024-11-26T23:25:44,682 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620, size=1.6 K (1623bytes) 2024-11-26T23:25:44,682 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620 2024-11-26T23:25:44,682 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620 after 0ms 2024-11-26T23:25:44,684 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620 took 3ms 2024-11-26T23:25:44,686 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620 so closing down 2024-11-26T23:25:44,686 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:44,688 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732663544620.temp 2024-11-26T23:25:44,689 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000003-wal.1732663544620.temp 2024-11-26T23:25:44,689 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:44,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741917_1095 (size=1623) 2024-11-26T23:25:44,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741917_1095 (size=1623) 2024-11-26T23:25:44,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741917_1095 (size=1623) 2024-11-26T23:25:44,696 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000003-wal.1732663544620.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:44,697 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000003-wal.1732663544620.temp to hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000012 2024-11-26T23:25:44,697 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-26T23:25:44,697 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620, journal: Splitting hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620, size=1.6 K (1623bytes) at 1732663544682Finishing writing output for hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620 so closing down at 1732663544686 (+4 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000003-wal.1732663544620.temp at 1732663544689 (+3 ms)3 split writer threads finished at 1732663544689Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000003-wal.1732663544620.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732663544696 (+7 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000003-wal.1732663544620.temp to hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000012 at 1732663544697 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1732663544697 2024-11-26T23:25:44,699 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663544620 to hdfs://localhost:45925/hbase/oldWALs/wal.1732663544620 2024-11-26T23:25:44,700 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000012 2024-11-26T23:25:44,908 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:25:45,009 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:45,011 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:45,025 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663545012, exclude list is [], retry=0 2024-11-26T23:25:45,028 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:45,028 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:45,028 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:45,030 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663545012 2024-11-26T23:25:45,030 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:45,030 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8acd1607604cbca6312ea4449ebda85e, NAME => 'testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:45,031 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:45,031 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,031 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,033 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,036 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8acd1607604cbca6312ea4449ebda85e columnFamilyName a 2024-11-26T23:25:45,036 DEBUG [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:45,037 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] regionserver.HStore(327): Store=8acd1607604cbca6312ea4449ebda85e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:45,037 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,038 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,039 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,040 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000012 2024-11-26T23:25:45,042 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000012 2024-11-26T23:25:45,043 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8acd1607604cbca6312ea4449ebda85e 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-26T23:25:45,061 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/.tmp/a/a24e11586a8f45a0823cc024c308a726 is 79, key is testDatalossWhenInputError/a:x0/1732663544653/Put/seqid=0 2024-11-26T23:25:45,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741919_1097 (size=5808) 2024-11-26T23:25:45,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741919_1097 (size=5808) 2024-11-26T23:25:45,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741919_1097 (size=5808) 2024-11-26T23:25:45,068 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/.tmp/a/a24e11586a8f45a0823cc024c308a726 2024-11-26T23:25:45,078 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/.tmp/a/a24e11586a8f45a0823cc024c308a726 as hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/a/a24e11586a8f45a0823cc024c308a726 2024-11-26T23:25:45,087 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/a/a24e11586a8f45a0823cc024c308a726, entries=10, sequenceid=12, filesize=5.7 K 2024-11-26T23:25:45,087 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 8acd1607604cbca6312ea4449ebda85e in 45ms, sequenceid=12, compaction requested=false; wal=null 2024-11-26T23:25:45,088 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/0000000000000000012 2024-11-26T23:25:45,089 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,089 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,091 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,093 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-26T23:25:45,094 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8acd1607604cbca6312ea4449ebda85e; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64875041, jitterRate=-0.03328655660152435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:25:45,095 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8acd1607604cbca6312ea4449ebda85e: Writing region info on filesystem at 1732663545031Initializing all the Stores at 1732663545033 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663545033Obtaining lock to block concurrent updates at 1732663545043 (+10 ms)Preparing flush snapshotting stores in 8acd1607604cbca6312ea4449ebda85e at 1732663545043Finished memstore snapshotting testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732663545043Flushing stores of testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e. at 1732663545043Flushing 8acd1607604cbca6312ea4449ebda85e/a: creating writer at 1732663545043Flushing 8acd1607604cbca6312ea4449ebda85e/a: appending metadata at 1732663545060 (+17 ms)Flushing 8acd1607604cbca6312ea4449ebda85e/a: closing flushed file at 1732663545060Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cdc6bc1: reopening flushed file at 1732663545076 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 8acd1607604cbca6312ea4449ebda85e in 45ms, sequenceid=12, compaction requested=false; wal=null at 1732663545087 (+11 ms)Cleaning up temporary data from old regions at 1732663545089 (+2 ms)Region opened successfully at 1732663545095 (+6 ms) 2024-11-26T23:25:45,098 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8acd1607604cbca6312ea4449ebda85e, NAME => 'testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:45,098 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732663544569.8acd1607604cbca6312ea4449ebda85e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:45,098 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,098 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,099 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,100 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8acd1607604cbca6312ea4449ebda85e columnFamilyName a 2024-11-26T23:25:45,100 DEBUG [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:45,105 DEBUG [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/a/a24e11586a8f45a0823cc024c308a726 2024-11-26T23:25:45,105 INFO [StoreOpener-8acd1607604cbca6312ea4449ebda85e-1 {}] regionserver.HStore(327): Store=8acd1607604cbca6312ea4449ebda85e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:45,105 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,106 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,107 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,107 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,107 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,109 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8acd1607604cbca6312ea4449ebda85e 2024-11-26T23:25:45,111 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testDatalossWhenInputError/8acd1607604cbca6312ea4449ebda85e/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-26T23:25:45,112 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8acd1607604cbca6312ea4449ebda85e; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62075391, jitterRate=-0.07500459253787994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:25:45,112 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8acd1607604cbca6312ea4449ebda85e: Writing region info on filesystem at 1732663545098Initializing all the Stores at 1732663545099 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663545099Cleaning up temporary data from old regions at 1732663545107 (+8 ms)Region opened successfully at 1732663545112 (+5 ms) 2024-11-26T23:25:45,133 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:57232 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:56914 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:57336 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:56998 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:60938 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1198857476-172.17.0.2-1732663512723:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:60898 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=276 (was 276), ProcessCount=11 (was 11), AvailableMemoryMB=6823 (was 6831) 2024-11-26T23:25:45,133 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-26T23:25:45,149 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=276, ProcessCount=11, AvailableMemoryMB=6823 2024-11-26T23:25:45,149 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-26T23:25:45,171 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:45,173 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:45,174 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:45,176 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82903656, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/hregion-82903656, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:45,195 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82903656/hregion-82903656.1732663545177, exclude list is [], retry=0 2024-11-26T23:25:45,198 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:45,198 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:45,199 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:45,202 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82903656/hregion-82903656.1732663545177 2024-11-26T23:25:45,204 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:45,204 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 41381fe1ffa7fe1907c3ab90051edf98, NAME => 'testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/hbase 2024-11-26T23:25:45,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741921_1099 (size=63) 2024-11-26T23:25:45,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741921_1099 (size=63) 2024-11-26T23:25:45,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741921_1099 (size=63) 2024-11-26T23:25:45,217 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:45,220 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,222 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41381fe1ffa7fe1907c3ab90051edf98 columnFamilyName a 2024-11-26T23:25:45,222 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:45,222 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(327): Store=41381fe1ffa7fe1907c3ab90051edf98/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:45,223 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,224 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41381fe1ffa7fe1907c3ab90051edf98 columnFamilyName b 2024-11-26T23:25:45,224 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:45,224 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(327): Store=41381fe1ffa7fe1907c3ab90051edf98/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:45,225 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,226 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41381fe1ffa7fe1907c3ab90051edf98 columnFamilyName c 2024-11-26T23:25:45,226 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:45,226 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(327): Store=41381fe1ffa7fe1907c3ab90051edf98/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:45,226 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,227 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,227 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,228 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,229 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,229 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:45,230 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,232 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:45,233 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41381fe1ffa7fe1907c3ab90051edf98; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63791084, jitterRate=-0.04943877458572388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:45,234 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41381fe1ffa7fe1907c3ab90051edf98: Writing region info on filesystem at 1732663545217Initializing all the Stores at 1732663545217Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663545217Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663545220 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663545220Cleaning up temporary data from old regions at 1732663545229 (+9 ms)Region opened successfully at 1732663545234 (+5 ms) 2024-11-26T23:25:45,234 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 41381fe1ffa7fe1907c3ab90051edf98, disabling compactions & flushes 2024-11-26T23:25:45,234 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. 2024-11-26T23:25:45,234 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. 2024-11-26T23:25:45,234 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. after waiting 0 ms 2024-11-26T23:25:45,234 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. 2024-11-26T23:25:45,235 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. 2024-11-26T23:25:45,235 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 41381fe1ffa7fe1907c3ab90051edf98: Waiting for close lock at 1732663545234Disabling compacts and flushes for region at 1732663545234Disabling writes for close at 1732663545234Writing region close event to WAL at 1732663545234Closed at 1732663545234 2024-11-26T23:25:45,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741920_1098 (size=93) 2024-11-26T23:25:45,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741920_1098 (size=93) 2024-11-26T23:25:45,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741920_1098 (size=93) 2024-11-26T23:25:45,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:45,244 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82903656:(num 1732663545177) 2024-11-26T23:25:45,245 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:45,246 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:45,260 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246, exclude list is [], retry=0 2024-11-26T23:25:45,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:45,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:45,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:45,265 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 2024-11-26T23:25:45,265 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:45,266 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 41381fe1ffa7fe1907c3ab90051edf98, NAME => 'testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:45,266 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:45,266 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,266 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,267 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,268 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41381fe1ffa7fe1907c3ab90051edf98 columnFamilyName a 2024-11-26T23:25:45,268 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:45,269 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(327): Store=41381fe1ffa7fe1907c3ab90051edf98/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:45,269 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,270 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41381fe1ffa7fe1907c3ab90051edf98 columnFamilyName b 2024-11-26T23:25:45,270 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:45,270 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(327): Store=41381fe1ffa7fe1907c3ab90051edf98/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:45,270 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,271 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41381fe1ffa7fe1907c3ab90051edf98 columnFamilyName c 2024-11-26T23:25:45,271 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:45,272 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(327): Store=41381fe1ffa7fe1907c3ab90051edf98/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:45,272 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,272 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,273 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,274 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,274 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,275 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:45,276 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:45,277 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41381fe1ffa7fe1907c3ab90051edf98; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60524192, jitterRate=-0.09811925888061523}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:45,277 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41381fe1ffa7fe1907c3ab90051edf98: Writing region info on filesystem at 1732663545266Initializing all the Stores at 1732663545267 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663545267Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663545267Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663545267Cleaning up temporary data from old regions at 1732663545274 (+7 ms)Region opened successfully at 1732663545277 (+3 ms) 2024-11-26T23:25:45,281 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732663545281/Put/seqid=0 2024-11-26T23:25:45,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741923_1101 (size=4875) 2024-11-26T23:25:45,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741923_1101 (size=4875) 2024-11-26T23:25:45,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741923_1101 (size=4875) 2024-11-26T23:25:45,293 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732663545293/Put/seqid=0 2024-11-26T23:25:45,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741924_1102 (size=4875) 2024-11-26T23:25:45,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741924_1102 (size=4875) 2024-11-26T23:25:45,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741924_1102 (size=4875) 2024-11-26T23:25:45,303 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732663545303/Put/seqid=0 2024-11-26T23:25:45,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741925_1103 (size=4875) 2024-11-26T23:25:45,315 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 41381fe1ffa7fe1907c3ab90051edf98/a 2024-11-26T23:25:45,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741925_1103 (size=4875) 2024-11-26T23:25:45,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741925_1103 (size=4875) 2024-11-26T23:25:45,319 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-26T23:25:45,319 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T23:25:45,320 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 41381fe1ffa7fe1907c3ab90051edf98/a 2024-11-26T23:25:45,323 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-26T23:25:45,323 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T23:25:45,324 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 41381fe1ffa7fe1907c3ab90051edf98/a 2024-11-26T23:25:45,327 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-26T23:25:45,327 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-26T23:25:45,327 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 41381fe1ffa7fe1907c3ab90051edf98 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-26T23:25:45,343 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/.tmp/a/086ca64779ac48358454a19bf7f74ad6 is 55, key is testCompactedBulkLoadedFiles/a:a/1732663545277/Put/seqid=0 2024-11-26T23:25:45,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741926_1104 (size=5107) 2024-11-26T23:25:45,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741926_1104 (size=5107) 2024-11-26T23:25:45,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741926_1104 (size=5107) 2024-11-26T23:25:45,350 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/.tmp/a/086ca64779ac48358454a19bf7f74ad6 2024-11-26T23:25:45,356 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/.tmp/a/086ca64779ac48358454a19bf7f74ad6 as hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/086ca64779ac48358454a19bf7f74ad6 2024-11-26T23:25:45,361 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/086ca64779ac48358454a19bf7f74ad6, entries=1, sequenceid=4, filesize=5.0 K 2024-11-26T23:25:45,362 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 41381fe1ffa7fe1907c3ab90051edf98 in 35ms, sequenceid=4, compaction requested=false 2024-11-26T23:25:45,362 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 41381fe1ffa7fe1907c3ab90051edf98: 2024-11-26T23:25:45,363 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/8f13d051dfe24808a405932428ac2905_SeqId_4_ 2024-11-26T23:25:45,364 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/2146f0a5b8f74486ab020b91296b4040_SeqId_4_ 2024-11-26T23:25:45,365 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_ 2024-11-26T23:25:45,366 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile0 into 41381fe1ffa7fe1907c3ab90051edf98/a as hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/8f13d051dfe24808a405932428ac2905_SeqId_4_ - updating store file list. 2024-11-26T23:25:45,370 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8f13d051dfe24808a405932428ac2905_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T23:25:45,370 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/8f13d051dfe24808a405932428ac2905_SeqId_4_ into 41381fe1ffa7fe1907c3ab90051edf98/a 2024-11-26T23:25:45,370 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile0 into 41381fe1ffa7fe1907c3ab90051edf98/a (new location: hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/8f13d051dfe24808a405932428ac2905_SeqId_4_) 2024-11-26T23:25:45,371 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile1 into 41381fe1ffa7fe1907c3ab90051edf98/a as hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/2146f0a5b8f74486ab020b91296b4040_SeqId_4_ - updating store file list. 2024-11-26T23:25:45,375 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2146f0a5b8f74486ab020b91296b4040_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T23:25:45,375 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/2146f0a5b8f74486ab020b91296b4040_SeqId_4_ into 41381fe1ffa7fe1907c3ab90051edf98/a 2024-11-26T23:25:45,375 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile1 into 41381fe1ffa7fe1907c3ab90051edf98/a (new location: hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/2146f0a5b8f74486ab020b91296b4040_SeqId_4_) 2024-11-26T23:25:45,376 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile2 into 41381fe1ffa7fe1907c3ab90051edf98/a as hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_ - updating store file list. 2024-11-26T23:25:45,379 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T23:25:45,379 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_ into 41381fe1ffa7fe1907c3ab90051edf98/a 2024-11-26T23:25:45,379 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:45925/hbase/testCompactedBulkLoadedFiles/hfile2 into 41381fe1ffa7fe1907c3ab90051edf98/a (new location: hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_) 2024-11-26T23:25:45,386 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-26T23:25:45,386 DEBUG [Time-limited test {}] regionserver.HStore(1541): 41381fe1ffa7fe1907c3ab90051edf98/a is initiating major compaction (all files) 2024-11-26T23:25:45,386 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 41381fe1ffa7fe1907c3ab90051edf98/a in testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. 2024-11-26T23:25:45,386 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/086ca64779ac48358454a19bf7f74ad6, hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/8f13d051dfe24808a405932428ac2905_SeqId_4_, hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/2146f0a5b8f74486ab020b91296b4040_SeqId_4_, hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_] into tmpdir=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/.tmp, totalSize=19.3 K 2024-11-26T23:25:45,387 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 086ca64779ac48358454a19bf7f74ad6, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732663545277 2024-11-26T23:25:45,387 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8f13d051dfe24808a405932428ac2905_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T23:25:45,387 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2146f0a5b8f74486ab020b91296b4040_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T23:25:45,388 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-26T23:25:45,399 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/.tmp/a/0ee5b12417de4b53a5978362dfebe575 is 55, key is testCompactedBulkLoadedFiles/a:a/1732663545277/Put/seqid=0 2024-11-26T23:25:45,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741927_1105 (size=6154) 2024-11-26T23:25:45,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741927_1105 (size=6154) 2024-11-26T23:25:45,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741927_1105 (size=6154) 2024-11-26T23:25:45,410 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/.tmp/a/0ee5b12417de4b53a5978362dfebe575 as hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/0ee5b12417de4b53a5978362dfebe575 2024-11-26T23:25:45,415 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 41381fe1ffa7fe1907c3ab90051edf98/a of 41381fe1ffa7fe1907c3ab90051edf98 into 0ee5b12417de4b53a5978362dfebe575(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:25:45,415 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 41381fe1ffa7fe1907c3ab90051edf98: 2024-11-26T23:25:45,416 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-26T23:25:45,416 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-26T23:25:45,449 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246, size=0 (0bytes) 2024-11-26T23:25:45,449 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 might be still open, length is 0 2024-11-26T23:25:45,449 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 2024-11-26T23:25:45,449 WARN [IPC Server handler 3 on default port 45925 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-26T23:25:45,450 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 after 0ms 2024-11-26T23:25:46,167 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-26T23:25:46,167 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:46,169 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-26T23:25:46,169 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:47,669 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:60978 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:36499:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60978 dst: /127.0.0.1:36499 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36499 remote=/127.0.0.1:60978]. Total timeout mills is 60000, 57747 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:47,669 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:57028 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43863:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57028 dst: /127.0.0.1:43863 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:47,669 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:57366 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37013:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57366 dst: /127.0.0.1:37013 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:47,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741922_1106 (size=1659) 2024-11-26T23:25:47,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741922_1106 (size=1659) 2024-11-26T23:25:47,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741922_1106 (size=1659) 2024-11-26T23:25:49,450 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 after 4001ms 2024-11-26T23:25:49,452 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:25:49,453 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 took 4004ms 2024-11-26T23:25:49,455 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246; continuing. 2024-11-26T23:25:49,455 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 so closing down 2024-11-26T23:25:49,455 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:49,457 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732663545246.temp 2024-11-26T23:25:49,459 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000003-wal.1732663545246.temp 2024-11-26T23:25:49,459 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:49,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741928_1107 (size=698) 2024-11-26T23:25:49,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741928_1107 (size=698) 2024-11-26T23:25:49,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741928_1107 (size=698) 2024-11-26T23:25:49,469 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000003-wal.1732663545246.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:49,470 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000003-wal.1732663545246.temp to hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000008 2024-11-26T23:25:49,470 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T23:25:49,471 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246, journal: Splitting hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246, size=0 (0bytes) at 1732663545449Finishing writing output for hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 so closing down at 1732663549455 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000003-wal.1732663545246.temp at 1732663549459 (+4 ms)3 split writer threads finished at 1732663549459Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000003-wal.1732663545246.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732663549469 (+10 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000003-wal.1732663545246.temp to hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000008 at 1732663549470 (+1 ms)Processed 5 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246, size=0, length=0, corrupted=false, cancelled=false at 1732663549470 2024-11-26T23:25:49,473 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 to hdfs://localhost:45925/hbase/oldWALs/wal.1732663545246 2024-11-26T23:25:49,474 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000008 2024-11-26T23:25:49,474 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:49,476 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:49,494 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663549476, exclude list is [], retry=0 2024-11-26T23:25:49,497 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:49,497 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:49,497 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:49,498 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663549476 2024-11-26T23:25:49,499 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:49,499 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 41381fe1ffa7fe1907c3ab90051edf98, NAME => 'testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:49,499 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:49,499 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,499 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,500 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,501 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41381fe1ffa7fe1907c3ab90051edf98 columnFamilyName a 2024-11-26T23:25:49,501 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,507 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/086ca64779ac48358454a19bf7f74ad6 2024-11-26T23:25:49,510 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/0ee5b12417de4b53a5978362dfebe575 2024-11-26T23:25:49,514 DEBUG [StoreFileOpener-41381fe1ffa7fe1907c3ab90051edf98-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2146f0a5b8f74486ab020b91296b4040_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T23:25:49,514 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/2146f0a5b8f74486ab020b91296b4040_SeqId_4_ 2024-11-26T23:25:49,518 DEBUG [StoreFileOpener-41381fe1ffa7fe1907c3ab90051edf98-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 8f13d051dfe24808a405932428ac2905_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T23:25:49,518 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/8f13d051dfe24808a405932428ac2905_SeqId_4_ 2024-11-26T23:25:49,521 DEBUG [StoreFileOpener-41381fe1ffa7fe1907c3ab90051edf98-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-26T23:25:49,521 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_ 2024-11-26T23:25:49,521 WARN [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/086ca64779ac48358454a19bf7f74ad6 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@74aeb39b 2024-11-26T23:25:49,521 WARN [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/2146f0a5b8f74486ab020b91296b4040_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@74aeb39b 2024-11-26T23:25:49,521 WARN [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/8f13d051dfe24808a405932428ac2905_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@74aeb39b 2024-11-26T23:25:49,521 WARN [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@74aeb39b 2024-11-26T23:25:49,522 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/086ca64779ac48358454a19bf7f74ad6, hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/2146f0a5b8f74486ab020b91296b4040_SeqId_4_, hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/8f13d051dfe24808a405932428ac2905_SeqId_4_, hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_] to archive 2024-11-26T23:25:49,522 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:25:49,524 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/086ca64779ac48358454a19bf7f74ad6 to hdfs://localhost:45925/hbase/archive/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/086ca64779ac48358454a19bf7f74ad6 2024-11-26T23:25:49,525 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/2146f0a5b8f74486ab020b91296b4040_SeqId_4_ to hdfs://localhost:45925/hbase/archive/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/2146f0a5b8f74486ab020b91296b4040_SeqId_4_ 2024-11-26T23:25:49,526 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/8f13d051dfe24808a405932428ac2905_SeqId_4_ to hdfs://localhost:45925/hbase/archive/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/8f13d051dfe24808a405932428ac2905_SeqId_4_ 2024-11-26T23:25:49,527 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_ to hdfs://localhost:45925/hbase/archive/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/a/950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_ 2024-11-26T23:25:49,528 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(327): Store=41381fe1ffa7fe1907c3ab90051edf98/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,528 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,529 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41381fe1ffa7fe1907c3ab90051edf98 columnFamilyName b 2024-11-26T23:25:49,529 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,529 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(327): Store=41381fe1ffa7fe1907c3ab90051edf98/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,529 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,530 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41381fe1ffa7fe1907c3ab90051edf98 columnFamilyName c 2024-11-26T23:25:49,530 DEBUG [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,531 INFO [StoreOpener-41381fe1ffa7fe1907c3ab90051edf98-1 {}] regionserver.HStore(327): Store=41381fe1ffa7fe1907c3ab90051edf98/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,531 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,531 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,533 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,533 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000008 2024-11-26T23:25:49,536 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 41381fe1ffa7fe1907c3ab90051edf98 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "41381fe1ffa7fe1907c3ab90051edf98" family_name: "a" compaction_input: "086ca64779ac48358454a19bf7f74ad6" compaction_input: "8f13d051dfe24808a405932428ac2905_SeqId_4_" compaction_input: "2146f0a5b8f74486ab020b91296b4040_SeqId_4_" compaction_input: "950210bc79ba4bc68bfaf9df0138baa2_SeqId_4_" compaction_output: "0ee5b12417de4b53a5978362dfebe575" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-26T23:25:49,536 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-26T23:25:49,536 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000008 2024-11-26T23:25:49,537 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/0000000000000000008 2024-11-26T23:25:49,538 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,538 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,539 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:49,540 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 41381fe1ffa7fe1907c3ab90051edf98 2024-11-26T23:25:49,542 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testCompactedBulkLoadedFiles/41381fe1ffa7fe1907c3ab90051edf98/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-26T23:25:49,543 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 41381fe1ffa7fe1907c3ab90051edf98; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62464700, jitterRate=-0.0692034363746643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:49,543 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 41381fe1ffa7fe1907c3ab90051edf98: Writing region info on filesystem at 1732663549499Initializing all the Stores at 1732663549500 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549500Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549500Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549500Cleaning up temporary data from old regions at 1732663549538 (+38 ms)Region opened successfully at 1732663549543 (+5 ms) 2024-11-26T23:25:49,545 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 41381fe1ffa7fe1907c3ab90051edf98, disabling compactions & flushes 2024-11-26T23:25:49,546 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. 2024-11-26T23:25:49,546 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. 2024-11-26T23:25:49,546 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. after waiting 0 ms 2024-11-26T23:25:49,546 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. 2024-11-26T23:25:49,546 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732663545172.41381fe1ffa7fe1907c3ab90051edf98. 2024-11-26T23:25:49,546 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 41381fe1ffa7fe1907c3ab90051edf98: Waiting for close lock at 1732663549545Disabling compacts and flushes for region at 1732663549545Disabling writes for close at 1732663549546 (+1 ms)Writing region close event to WAL at 1732663549546Closed at 1732663549546 2024-11-26T23:25:49,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741929_1108 (size=93) 2024-11-26T23:25:49,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741929_1108 (size=93) 2024-11-26T23:25:49,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741929_1108 (size=93) 2024-11-26T23:25:49,551 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:49,551 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732663549476) 2024-11-26T23:25:49,568 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=438 (was 425) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1576013078_22 at /127.0.0.1:57102 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1576013078_22 at /127.0.0.1:57430 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1576013078_22 at /127.0.0.1:32800 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:45925 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:45925 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 276), ProcessCount=11 (was 11), AvailableMemoryMB=6809 (was 6823) 2024-11-26T23:25:49,568 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-26T23:25:49,583 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=253, ProcessCount=11, AvailableMemoryMB=6807 2024-11-26T23:25:49,583 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-26T23:25:49,598 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:49,600 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T23:25:49,600 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T23:25:49,603 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-75557609, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/hregion-75557609, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:49,619 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-75557609/hregion-75557609.1732663549603, exclude list is [], retry=0 2024-11-26T23:25:49,622 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:49,622 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:49,622 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:49,624 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-75557609/hregion-75557609.1732663549603 2024-11-26T23:25:49,624 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819)] 2024-11-26T23:25:49,625 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 888e9af0ad40706b69f303bbf24d09ec, NAME => 'testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45925/hbase 2024-11-26T23:25:49,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741931_1110 (size=67) 2024-11-26T23:25:49,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741931_1110 (size=67) 2024-11-26T23:25:49,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741931_1110 (size=67) 2024-11-26T23:25:49,634 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:49,635 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,636 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName a 2024-11-26T23:25:49,636 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,637 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,637 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,638 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName b 2024-11-26T23:25:49,638 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,639 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,639 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,640 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName c 2024-11-26T23:25:49,640 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,640 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,640 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,641 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,642 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,643 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,643 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,643 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:49,644 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,646 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:25:49,647 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 888e9af0ad40706b69f303bbf24d09ec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63803966, jitterRate=-0.04924681782722473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:49,647 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 888e9af0ad40706b69f303bbf24d09ec: Writing region info on filesystem at 1732663549634Initializing all the Stores at 1732663549635 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549635Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549635Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549635Cleaning up temporary data from old regions at 1732663549643 (+8 ms)Region opened successfully at 1732663549647 (+4 ms) 2024-11-26T23:25:49,647 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 888e9af0ad40706b69f303bbf24d09ec, disabling compactions & flushes 2024-11-26T23:25:49,647 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:49,647 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:49,647 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. after waiting 0 ms 2024-11-26T23:25:49,647 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:49,647 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:49,647 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 888e9af0ad40706b69f303bbf24d09ec: Waiting for close lock at 1732663549647Disabling compacts and flushes for region at 1732663549647Disabling writes for close at 1732663549647Writing region close event to WAL at 1732663549647Closed at 1732663549647 2024-11-26T23:25:49,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741930_1109 (size=93) 2024-11-26T23:25:49,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741930_1109 (size=93) 2024-11-26T23:25:49,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741930_1109 (size=93) 2024-11-26T23:25:49,653 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:49,653 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-75557609:(num 1732663549603) 2024-11-26T23:25:49,653 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:49,654 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:49,668 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655, exclude list is [], retry=0 2024-11-26T23:25:49,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:49,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:49,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:49,673 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655 2024-11-26T23:25:49,673 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:42789:42789)] 2024-11-26T23:25:49,673 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 888e9af0ad40706b69f303bbf24d09ec, NAME => 'testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:49,673 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:49,673 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,673 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,675 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,676 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName a 2024-11-26T23:25:49,676 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,676 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,677 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,677 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName b 2024-11-26T23:25:49,677 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,678 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,678 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,679 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName c 2024-11-26T23:25:49,679 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,679 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,679 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,680 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,681 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,682 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,682 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,682 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:49,683 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,684 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 888e9af0ad40706b69f303bbf24d09ec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59273294, jitterRate=-0.11675909161567688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:49,684 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 888e9af0ad40706b69f303bbf24d09ec: Writing region info on filesystem at 1732663549674Initializing all the Stores at 1732663549674Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549674Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549675 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549675Cleaning up temporary data from old regions at 1732663549682 (+7 ms)Region opened successfully at 1732663549684 (+2 ms) 2024-11-26T23:25:49,691 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 888e9af0ad40706b69f303bbf24d09ec 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-26T23:25:49,707 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/a/10c60222227d4fc89eb1a040c70f630a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732663549684/Put/seqid=0 2024-11-26T23:25:49,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741933_1112 (size=5958) 2024-11-26T23:25:49,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741933_1112 (size=5958) 2024-11-26T23:25:49,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741933_1112 (size=5958) 2024-11-26T23:25:49,716 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/a/10c60222227d4fc89eb1a040c70f630a 2024-11-26T23:25:49,722 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/a/10c60222227d4fc89eb1a040c70f630a as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/a/10c60222227d4fc89eb1a040c70f630a 2024-11-26T23:25:49,727 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/a/10c60222227d4fc89eb1a040c70f630a, entries=10, sequenceid=13, filesize=5.8 K 2024-11-26T23:25:49,728 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 888e9af0ad40706b69f303bbf24d09ec in 37ms, sequenceid=13, compaction requested=false 2024-11-26T23:25:49,728 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 888e9af0ad40706b69f303bbf24d09ec: 2024-11-26T23:25:49,745 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 888e9af0ad40706b69f303bbf24d09ec, disabling compactions & flushes 2024-11-26T23:25:49,745 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:49,745 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:49,745 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. after waiting 0 ms 2024-11-26T23:25:49,745 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:49,746 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:49,746 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:49,746 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 888e9af0ad40706b69f303bbf24d09ec: Waiting for close lock at 1732663549745Disabling compacts and flushes for region at 1732663549745Disabling writes for close at 1732663549745Writing region close event to WAL at 1732663549746 (+1 ms)Closed at 1732663549746 2024-11-26T23:25:49,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741932_1111 (size=5869) 2024-11-26T23:25:49,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741932_1111 (size=5869) 2024-11-26T23:25:49,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741932_1111 (size=5869) 2024-11-26T23:25:49,765 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655, size=5.7 K (5869bytes) 2024-11-26T23:25:49,765 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655 2024-11-26T23:25:49,766 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655 after 1ms 2024-11-26T23:25:49,768 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655 took 3ms 2024-11-26T23:25:49,770 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655 so closing down 2024-11-26T23:25:49,770 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:49,771 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732663549655.temp 2024-11-26T23:25:49,772 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000003-wal.1732663549655.temp 2024-11-26T23:25:49,772 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:49,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741934_1113 (size=5223) 2024-11-26T23:25:49,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741934_1113 (size=5223) 2024-11-26T23:25:49,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741934_1113 (size=5223) 2024-11-26T23:25:49,779 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000003-wal.1732663549655.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:49,780 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000003-wal.1732663549655.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000035 2024-11-26T23:25:49,780 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-26T23:25:49,780 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655, journal: Splitting hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655, size=5.7 K (5869bytes) at 1732663549765Finishing writing output for hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655 so closing down at 1732663549770 (+5 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000003-wal.1732663549655.temp at 1732663549772 (+2 ms)3 split writer threads finished at 1732663549772Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000003-wal.1732663549655.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732663549779 (+7 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000003-wal.1732663549655.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000035 at 1732663549780 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1732663549780 2024-11-26T23:25:49,781 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549655 to hdfs://localhost:45925/hbase/oldWALs/wal.1732663549655 2024-11-26T23:25:49,782 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000035 2024-11-26T23:25:49,782 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:49,784 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:49,797 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784, exclude list is [], retry=0 2024-11-26T23:25:49,799 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:49,800 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:49,800 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:49,801 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 2024-11-26T23:25:49,802 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:49,802 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 888e9af0ad40706b69f303bbf24d09ec, NAME => 'testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:25:49,802 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:49,802 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,802 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,803 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,804 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName a 2024-11-26T23:25:49,804 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,809 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/a/10c60222227d4fc89eb1a040c70f630a 2024-11-26T23:25:49,809 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,810 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,810 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName b 2024-11-26T23:25:49,811 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,811 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,811 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,812 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName c 2024-11-26T23:25:49,812 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:49,813 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:49,813 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,814 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,815 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,815 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000035 2024-11-26T23:25:49,818 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000035 2024-11-26T23:25:49,818 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 888e9af0ad40706b69f303bbf24d09ec 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-26T23:25:49,834 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/b/3bf2899fd4c44847ba7f674aa9074bfa is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732663549728/Put/seqid=0 2024-11-26T23:25:49,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741936_1115 (size=5958) 2024-11-26T23:25:49,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741936_1115 (size=5958) 2024-11-26T23:25:49,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741936_1115 (size=5958) 2024-11-26T23:25:49,841 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/b/3bf2899fd4c44847ba7f674aa9074bfa 2024-11-26T23:25:49,860 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/c/704d768fd1a245f19736d0657c10b43d is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732663549735/Put/seqid=0 2024-11-26T23:25:49,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741937_1116 (size=5958) 2024-11-26T23:25:49,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741937_1116 (size=5958) 2024-11-26T23:25:49,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741937_1116 (size=5958) 2024-11-26T23:25:49,867 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/c/704d768fd1a245f19736d0657c10b43d 2024-11-26T23:25:49,872 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/b/3bf2899fd4c44847ba7f674aa9074bfa as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/b/3bf2899fd4c44847ba7f674aa9074bfa 2024-11-26T23:25:49,876 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/b/3bf2899fd4c44847ba7f674aa9074bfa, entries=10, sequenceid=35, filesize=5.8 K 2024-11-26T23:25:49,877 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/c/704d768fd1a245f19736d0657c10b43d as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/c/704d768fd1a245f19736d0657c10b43d 2024-11-26T23:25:49,881 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/c/704d768fd1a245f19736d0657c10b43d, entries=10, sequenceid=35, filesize=5.8 K 2024-11-26T23:25:49,882 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 888e9af0ad40706b69f303bbf24d09ec in 63ms, sequenceid=35, compaction requested=false; wal=null 2024-11-26T23:25:49,882 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000035 2024-11-26T23:25:49,883 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,883 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,884 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:49,885 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:49,887 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-26T23:25:49,887 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 888e9af0ad40706b69f303bbf24d09ec; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61257369, jitterRate=-0.08719407021999359}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:49,888 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 888e9af0ad40706b69f303bbf24d09ec: Writing region info on filesystem at 1732663549802Initializing all the Stores at 1732663549803 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549803Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549803Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663549803Obtaining lock to block concurrent updates at 1732663549818 (+15 ms)Preparing flush snapshotting stores in 888e9af0ad40706b69f303bbf24d09ec at 1732663549818Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732663549819 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. at 1732663549819Flushing 888e9af0ad40706b69f303bbf24d09ec/b: creating writer at 1732663549819Flushing 888e9af0ad40706b69f303bbf24d09ec/b: appending metadata at 1732663549833 (+14 ms)Flushing 888e9af0ad40706b69f303bbf24d09ec/b: closing flushed file at 1732663549833Flushing 888e9af0ad40706b69f303bbf24d09ec/c: creating writer at 1732663549845 (+12 ms)Flushing 888e9af0ad40706b69f303bbf24d09ec/c: appending metadata at 1732663549859 (+14 ms)Flushing 888e9af0ad40706b69f303bbf24d09ec/c: closing flushed file at 1732663549859Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@659265b5: reopening flushed file at 1732663549871 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7675bed7: reopening flushed file at 1732663549876 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 888e9af0ad40706b69f303bbf24d09ec in 63ms, sequenceid=35, compaction requested=false; wal=null at 1732663549882 (+6 ms)Cleaning up temporary data from old regions at 1732663549883 (+1 ms)Region opened successfully at 1732663549888 (+5 ms) 2024-11-26T23:25:49,959 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784, size=0 (0bytes) 2024-11-26T23:25:49,959 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 might be still open, length is 0 2024-11-26T23:25:49,959 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 2024-11-26T23:25:49,959 WARN [IPC Server handler 1 on default port 45925 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-26T23:25:49,960 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 after 1ms 2024-11-26T23:25:50,669 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:57482 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37013:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57482 dst: /127.0.0.1:37013 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37013 remote=/127.0.0.1:57482]. Total timeout mills is 60000, 59245 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:50,670 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:57176 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43863:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57176 dst: /127.0.0.1:43863 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:50,670 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1750284896_22 at /127.0.0.1:32846 [Receiving block BP-1198857476-172.17.0.2-1732663512723:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:36499:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:32846 dst: /127.0.0.1:36499 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:50,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741935_1117 (size=5215) 2024-11-26T23:25:50,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741935_1117 (size=5215) 2024-11-26T23:25:50,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741935_1117 (size=5215) 2024-11-26T23:25:53,960 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 after 4001ms 2024-11-26T23:25:53,963 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 took 4004ms 2024-11-26T23:25:53,965 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784; continuing. 2024-11-26T23:25:53,965 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 so closing down 2024-11-26T23:25:53,965 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T23:25:53,966 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732663549784.temp 2024-11-26T23:25:53,968 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000037-wal.1732663549784.temp 2024-11-26T23:25:53,968 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T23:25:53,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741938_1118 (size=5223) 2024-11-26T23:25:53,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741938_1118 (size=5223) 2024-11-26T23:25:53,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741938_1118 (size=5223) 2024-11-26T23:25:53,974 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000037-wal.1732663549784.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-26T23:25:53,976 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000037-wal.1732663549784.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000066 2024-11-26T23:25:53,976 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784, size=0, length=0, corrupted=false, cancelled=false 2024-11-26T23:25:53,976 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784, journal: Splitting hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784, size=0 (0bytes) at 1732663549959Finishing writing output for hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 so closing down at 1732663553965 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000037-wal.1732663549784.temp at 1732663553968 (+3 ms)3 split writer threads finished at 1732663553968Closed recovered edits writer path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000037-wal.1732663549784.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732663553974 (+6 ms)Rename recovered edits hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000037-wal.1732663549784.temp to hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000066 at 1732663553976 (+2 ms)Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784, size=0, length=0, corrupted=false, cancelled=false at 1732663553976 2024-11-26T23:25:53,977 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 to hdfs://localhost:45925/hbase/oldWALs/wal.1732663549784 2024-11-26T23:25:53,978 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000066 2024-11-26T23:25:53,978 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-26T23:25:53,980 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:45925/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597, archiveDir=hdfs://localhost:45925/hbase/oldWALs, maxLogs=32 2024-11-26T23:25:53,993 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663553980, exclude list is [], retry=0 2024-11-26T23:25:53,995 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37013,DS-13793ef1-03f2-4b22-a065-26d2a150e69d,DISK] 2024-11-26T23:25:53,996 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36499,DS-f976893c-cf48-4bea-904b-c9f351c04507,DISK] 2024-11-26T23:25:53,996 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43863,DS-99717455-0f20-44e5-9444-1c9bdbc35fbb,DISK] 2024-11-26T23:25:53,997 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663553980 2024-11-26T23:25:53,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42789:42789),(127.0.0.1/127.0.0.1:41819:41819),(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-26T23:25:53,999 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:25:54,000 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:54,001 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName a 2024-11-26T23:25:54,001 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:54,006 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/a/10c60222227d4fc89eb1a040c70f630a 2024-11-26T23:25:54,007 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:54,007 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:54,008 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName b 2024-11-26T23:25:54,008 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:54,012 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/b/3bf2899fd4c44847ba7f674aa9074bfa 2024-11-26T23:25:54,012 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:54,012 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:54,013 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 888e9af0ad40706b69f303bbf24d09ec columnFamilyName c 2024-11-26T23:25:54,013 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:25:54,018 DEBUG [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/c/704d768fd1a245f19736d0657c10b43d 2024-11-26T23:25:54,018 INFO [StoreOpener-888e9af0ad40706b69f303bbf24d09ec-1 {}] regionserver.HStore(327): Store=888e9af0ad40706b69f303bbf24d09ec/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:25:54,018 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:54,019 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:54,020 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:54,020 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000066 2024-11-26T23:25:54,027 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000066 2024-11-26T23:25:54,027 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 888e9af0ad40706b69f303bbf24d09ec 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-26T23:25:54,042 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/a/3e3cfec9c10b482e8749b7f6a53d5a7c is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732663549894/Put/seqid=0 2024-11-26T23:25:54,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741940_1120 (size=5958) 2024-11-26T23:25:54,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741940_1120 (size=5958) 2024-11-26T23:25:54,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741940_1120 (size=5958) 2024-11-26T23:25:54,049 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/a/3e3cfec9c10b482e8749b7f6a53d5a7c 2024-11-26T23:25:54,068 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/b/ee3d6e35779b468aad52d15e2b0562d7 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732663549901/Put/seqid=0 2024-11-26T23:25:54,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741941_1121 (size=5958) 2024-11-26T23:25:54,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741941_1121 (size=5958) 2024-11-26T23:25:54,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741941_1121 (size=5958) 2024-11-26T23:25:54,077 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/b/ee3d6e35779b468aad52d15e2b0562d7 2024-11-26T23:25:54,095 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/c/1738ed2d94d1460895c746215dbb8a41 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732663549907/Put/seqid=0 2024-11-26T23:25:54,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741942_1122 (size=5958) 2024-11-26T23:25:54,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741942_1122 (size=5958) 2024-11-26T23:25:54,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741942_1122 (size=5958) 2024-11-26T23:25:54,101 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/c/1738ed2d94d1460895c746215dbb8a41 2024-11-26T23:25:54,106 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/a/3e3cfec9c10b482e8749b7f6a53d5a7c as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/a/3e3cfec9c10b482e8749b7f6a53d5a7c 2024-11-26T23:25:54,110 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/a/3e3cfec9c10b482e8749b7f6a53d5a7c, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T23:25:54,111 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/b/ee3d6e35779b468aad52d15e2b0562d7 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/b/ee3d6e35779b468aad52d15e2b0562d7 2024-11-26T23:25:54,116 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/b/ee3d6e35779b468aad52d15e2b0562d7, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T23:25:54,116 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/.tmp/c/1738ed2d94d1460895c746215dbb8a41 as hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/c/1738ed2d94d1460895c746215dbb8a41 2024-11-26T23:25:54,121 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/c/1738ed2d94d1460895c746215dbb8a41, entries=10, sequenceid=66, filesize=5.8 K 2024-11-26T23:25:54,121 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 888e9af0ad40706b69f303bbf24d09ec in 94ms, sequenceid=66, compaction requested=false; wal=null 2024-11-26T23:25:54,122 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/0000000000000000066 2024-11-26T23:25:54,123 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:54,123 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:54,123 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-26T23:25:54,124 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 888e9af0ad40706b69f303bbf24d09ec 2024-11-26T23:25:54,126 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/hbase/data/default/testReplayEditsWrittenViaHRegion/888e9af0ad40706b69f303bbf24d09ec/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-26T23:25:54,127 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 888e9af0ad40706b69f303bbf24d09ec; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70739513, jitterRate=0.0541008859872818}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-26T23:25:54,127 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 888e9af0ad40706b69f303bbf24d09ec: Writing region info on filesystem at 1732663553999Initializing all the Stores at 1732663554000 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663554000Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663554000Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663554000Obtaining lock to block concurrent updates at 1732663554027 (+27 ms)Preparing flush snapshotting stores in 888e9af0ad40706b69f303bbf24d09ec at 1732663554027Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732663554027Flushing stores of testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. at 1732663554027Flushing 888e9af0ad40706b69f303bbf24d09ec/a: creating writer at 1732663554027Flushing 888e9af0ad40706b69f303bbf24d09ec/a: appending metadata at 1732663554041 (+14 ms)Flushing 888e9af0ad40706b69f303bbf24d09ec/a: closing flushed file at 1732663554041Flushing 888e9af0ad40706b69f303bbf24d09ec/b: creating writer at 1732663554053 (+12 ms)Flushing 888e9af0ad40706b69f303bbf24d09ec/b: appending metadata at 1732663554067 (+14 ms)Flushing 888e9af0ad40706b69f303bbf24d09ec/b: closing flushed file at 1732663554067Flushing 888e9af0ad40706b69f303bbf24d09ec/c: creating writer at 1732663554081 (+14 ms)Flushing 888e9af0ad40706b69f303bbf24d09ec/c: appending metadata at 1732663554094 (+13 ms)Flushing 888e9af0ad40706b69f303bbf24d09ec/c: closing flushed file at 1732663554095 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c5d060: reopening flushed file at 1732663554105 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7306d7d5: reopening flushed file at 1732663554111 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bac1911: reopening flushed file at 1732663554116 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 888e9af0ad40706b69f303bbf24d09ec in 94ms, sequenceid=66, compaction requested=false; wal=null at 1732663554121 (+5 ms)Cleaning up temporary data from old regions at 1732663554123 (+2 ms)Region opened successfully at 1732663554127 (+4 ms) 2024-11-26T23:25:54,139 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 888e9af0ad40706b69f303bbf24d09ec, disabling compactions & flushes 2024-11-26T23:25:54,139 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:54,139 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:54,139 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. after waiting 0 ms 2024-11-26T23:25:54,139 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:54,141 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732663549598.888e9af0ad40706b69f303bbf24d09ec. 2024-11-26T23:25:54,141 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 888e9af0ad40706b69f303bbf24d09ec: Waiting for close lock at 1732663554139Disabling compacts and flushes for region at 1732663554139Disabling writes for close at 1732663554139Writing region close event to WAL at 1732663554141 (+2 ms)Closed at 1732663554141 2024-11-26T23:25:54,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741939_1119 (size=93) 2024-11-26T23:25:54,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741939_1119 (size=93) 2024-11-26T23:25:54,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741939_1119 (size=93) 2024-11-26T23:25:54,145 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-26T23:25:54,145 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732663553980) 2024-11-26T23:25:54,163 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=445 (was 438) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1196637763_22 at /127.0.0.1:51534 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1196637763_22 at /127.0.0.1:51940 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:45925 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:45925 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1196637763_22 at /127.0.0.1:56446 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=253 (was 253), ProcessCount=11 (was 11), AvailableMemoryMB=6800 (was 6807) 2024-11-26T23:25:54,164 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-26T23:25:54,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T23:25:54,164 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:25:54,164 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:25:54,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:25:54,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:25:54,165 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:25:54,165 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T23:25:54,165 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=16626632, stopped=false 2024-11-26T23:25:54,165 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c707e9ace675,35437,1732663515757 2024-11-26T23:25:54,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:25:54,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:25:54,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:25:54,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:54,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:54,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:54,167 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:25:54,168 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:25:54,168 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:25:54,168 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:25:54,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:25:54,168 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:25:54,168 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,34663,1732663516528' ***** 2024-11-26T23:25:54,168 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:25:54,168 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,46791,1732663516570' ***** 2024-11-26T23:25:54,168 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:25:54,168 INFO [RS:1;c707e9ace675:34663 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:25:54,169 INFO [RS:1;c707e9ace675:34663 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:25:54,169 INFO [RS:1;c707e9ace675:34663 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:25:54,169 INFO [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,34663,1732663516528 2024-11-26T23:25:54,169 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:25:54,169 INFO [RS:1;c707e9ace675:34663 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:25:54,169 INFO [RS:1;c707e9ace675:34663 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c707e9ace675:34663. 2024-11-26T23:25:54,169 INFO [RS:2;c707e9ace675:46791 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:25:54,169 DEBUG [RS:1;c707e9ace675:34663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:25:54,169 INFO [RS:2;c707e9ace675:46791 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:25:54,169 DEBUG [RS:1;c707e9ace675:34663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:25:54,169 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:54,169 INFO [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,34663,1732663516528; all regions closed. 2024-11-26T23:25:54,169 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:25:54,169 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,46791,1732663516570 2024-11-26T23:25:54,169 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:25:54,169 INFO [RS:2;c707e9ace675:46791 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:25:54,169 INFO [RS:2;c707e9ace675:46791 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;c707e9ace675:46791. 2024-11-26T23:25:54,170 DEBUG [RS:2;c707e9ace675:46791 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:25:54,170 DEBUG [RS:2;c707e9ace675:46791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:25:54,170 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c0071604d340e73f1eaa3d5a4b74f1f4, disabling compactions & flushes 2024-11-26T23:25:54,170 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:54,170 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:25:54,170 INFO [RS:2;c707e9ace675:46791 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:25:54,170 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:54,170 INFO [RS:2;c707e9ace675:46791 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:25:54,170 INFO [RS:2;c707e9ace675:46791 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:25:54,170 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. after waiting 0 ms 2024-11-26T23:25:54,170 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:54,170 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T23:25:54,171 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T23:25:54,171 DEBUG [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, c0071604d340e73f1eaa3d5a4b74f1f4=testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4.} 2024-11-26T23:25:54,171 DEBUG [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c0071604d340e73f1eaa3d5a4b74f1f4 2024-11-26T23:25:54,171 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:25:54,171 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:25:54,171 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:25:54,171 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:25:54,171 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:25:54,171 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-26T23:25:54,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741834_1010 (size=2598) 2024-11-26T23:25:54,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741834_1010 (size=2598) 2024-11-26T23:25:54,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741834_1010 (size=2598) 2024-11-26T23:25:54,174 DEBUG [RS:1;c707e9ace675:34663 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/oldWALs 2024-11-26T23:25:54,174 INFO [RS:1;c707e9ace675:34663 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c707e9ace675%2C34663%2C1732663516528:(num 1732663517893) 2024-11-26T23:25:54,174 DEBUG [RS:1;c707e9ace675:34663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:25:54,174 INFO [RS:1;c707e9ace675:34663 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:25:54,175 INFO [RS:1;c707e9ace675:34663 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:25:54,175 INFO [RS:1;c707e9ace675:34663 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T23:25:54,175 INFO [RS:1;c707e9ace675:34663 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:25:54,175 INFO [RS:1;c707e9ace675:34663 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:25:54,175 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:25:54,175 INFO [RS:1;c707e9ace675:34663 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:25:54,175 INFO [RS:1;c707e9ace675:34663 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:25:54,176 INFO [RS:1;c707e9ace675:34663 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34663 2024-11-26T23:25:54,177 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/default/testReplayEditsAfterRegionMovedWithMultiCF/c0071604d340e73f1eaa3d5a4b74f1f4/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-26T23:25:54,178 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/info/647116e460f842c0b46e867d4779cb69 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4./info:regioninfo/1732663534057/Put/seqid=0 2024-11-26T23:25:54,178 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:54,178 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c0071604d340e73f1eaa3d5a4b74f1f4: Waiting for close lock at 1732663554169Running coprocessor pre-close hooks at 1732663554169Disabling compacts and flushes for region at 1732663554169Disabling writes for close at 1732663554170 (+1 ms)Writing region close event to WAL at 1732663554174 (+4 ms)Running coprocessor post-close hooks at 1732663554178 (+4 ms)Closed at 1732663554178 2024-11-26T23:25:54,178 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732663530227.c0071604d340e73f1eaa3d5a4b74f1f4. 2024-11-26T23:25:54,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:25:54,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,34663,1732663516528 2024-11-26T23:25:54,179 INFO [RS:1;c707e9ace675:34663 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:25:54,179 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,34663,1732663516528] 2024-11-26T23:25:54,181 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,34663,1732663516528 already deleted, retry=false 2024-11-26T23:25:54,182 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,34663,1732663516528 expired; onlineServers=1 2024-11-26T23:25:54,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741943_1123 (size=6778) 2024-11-26T23:25:54,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741943_1123 (size=6778) 2024-11-26T23:25:54,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741943_1123 (size=6778) 2024-11-26T23:25:54,184 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/info/647116e460f842c0b46e867d4779cb69 2024-11-26T23:25:54,190 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/.tmp/info/647116e460f842c0b46e867d4779cb69 as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/info/647116e460f842c0b46e867d4779cb69 2024-11-26T23:25:54,194 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/info/647116e460f842c0b46e867d4779cb69, entries=8, sequenceid=23, filesize=6.6 K 2024-11-26T23:25:54,195 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false 2024-11-26T23:25:54,199 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-26T23:25:54,200 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:25:54,200 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:25:54,200 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663554171Running coprocessor pre-close hooks at 1732663554171Disabling compacts and flushes for region at 1732663554171Disabling writes for close at 1732663554171Obtaining lock to block concurrent updates at 1732663554171Preparing flush snapshotting stores in 1588230740 at 1732663554171Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732663554172 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732663554172Flushing 1588230740/info: creating writer at 1732663554173 (+1 ms)Flushing 1588230740/info: appending metadata at 1732663554177 (+4 ms)Flushing 1588230740/info: closing flushed file at 1732663554177Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20e921c2: reopening flushed file at 1732663554189 (+12 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false at 1732663554195 (+6 ms)Writing region close event to WAL at 1732663554196 (+1 ms)Running coprocessor post-close hooks at 1732663554199 (+3 ms)Closed at 1732663554200 (+1 ms) 2024-11-26T23:25:54,200 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T23:25:54,253 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:25:54,253 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:25:54,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:25:54,281 INFO [RS:1;c707e9ace675:34663 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:25:54,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34663-0x101569901eb0002, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:25:54,281 INFO [RS:1;c707e9ace675:34663 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,34663,1732663516528; zookeeper connection closed. 2024-11-26T23:25:54,281 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b15c620 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b15c620 2024-11-26T23:25:54,371 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,46791,1732663516570; all regions closed. 2024-11-26T23:25:54,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741893_1071 (size=2422) 2024-11-26T23:25:54,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741893_1071 (size=2422) 2024-11-26T23:25:54,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741893_1071 (size=2422) 2024-11-26T23:25:54,376 DEBUG [RS:2;c707e9ace675:46791 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/oldWALs 2024-11-26T23:25:54,376 INFO [RS:2;c707e9ace675:46791 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c707e9ace675%2C46791%2C1732663516570.meta:.meta(num 1732663533241) 2024-11-26T23:25:54,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741835_1011 (size=928) 2024-11-26T23:25:54,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741835_1011 (size=928) 2024-11-26T23:25:54,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741835_1011 (size=928) 2024-11-26T23:25:54,380 DEBUG [RS:2;c707e9ace675:46791 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/oldWALs 2024-11-26T23:25:54,380 INFO [RS:2;c707e9ace675:46791 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c707e9ace675%2C46791%2C1732663516570:(num 1732663517893) 2024-11-26T23:25:54,380 DEBUG [RS:2;c707e9ace675:46791 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:25:54,380 INFO [RS:2;c707e9ace675:46791 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:25:54,380 INFO [RS:2;c707e9ace675:46791 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:25:54,380 INFO [RS:2;c707e9ace675:46791 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T23:25:54,381 INFO [RS:2;c707e9ace675:46791 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:25:54,381 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:25:54,381 INFO [RS:2;c707e9ace675:46791 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46791 2024-11-26T23:25:54,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,46791,1732663516570 2024-11-26T23:25:54,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:25:54,383 INFO [RS:2;c707e9ace675:46791 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:25:54,391 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,46791,1732663516570] 2024-11-26T23:25:54,392 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,46791,1732663516570 already deleted, retry=false 2024-11-26T23:25:54,392 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,46791,1732663516570 expired; onlineServers=0 2024-11-26T23:25:54,392 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c707e9ace675,35437,1732663515757' ***** 2024-11-26T23:25:54,392 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T23:25:54,392 INFO [M:0;c707e9ace675:35437 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:25:54,392 INFO [M:0;c707e9ace675:35437 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:25:54,392 DEBUG [M:0;c707e9ace675:35437 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T23:25:54,392 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T23:25:54,392 DEBUG [M:0;c707e9ace675:35437 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T23:25:54,392 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663517551 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663517551,5,FailOnTimeoutGroup] 2024-11-26T23:25:54,392 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663517553 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663517553,5,FailOnTimeoutGroup] 2024-11-26T23:25:54,393 INFO [M:0;c707e9ace675:35437 {}] hbase.ChoreService(370): Chore service for: master/c707e9ace675:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T23:25:54,393 INFO [M:0;c707e9ace675:35437 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:25:54,393 DEBUG [M:0;c707e9ace675:35437 {}] master.HMaster(1795): Stopping service threads 2024-11-26T23:25:54,393 INFO [M:0;c707e9ace675:35437 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T23:25:54,393 INFO [M:0;c707e9ace675:35437 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:25:54,394 INFO [M:0;c707e9ace675:35437 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T23:25:54,394 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T23:25:54,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T23:25:54,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:25:54,394 DEBUG [M:0;c707e9ace675:35437 {}] zookeeper.ZKUtil(347): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T23:25:54,394 WARN [M:0;c707e9ace675:35437 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T23:25:54,395 INFO [M:0;c707e9ace675:35437 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/.lastflushedseqids 2024-11-26T23:25:54,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741944_1124 (size=119) 2024-11-26T23:25:54,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741944_1124 (size=119) 2024-11-26T23:25:54,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741944_1124 (size=119) 2024-11-26T23:25:54,432 INFO [M:0;c707e9ace675:35437 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T23:25:54,432 INFO [M:0;c707e9ace675:35437 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T23:25:54,433 DEBUG [M:0;c707e9ace675:35437 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:25:54,433 INFO [M:0;c707e9ace675:35437 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:25:54,433 DEBUG [M:0;c707e9ace675:35437 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:25:54,433 DEBUG [M:0;c707e9ace675:35437 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:25:54,433 DEBUG [M:0;c707e9ace675:35437 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:25:54,433 INFO [M:0;c707e9ace675:35437 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-11-26T23:25:54,452 DEBUG [M:0;c707e9ace675:35437 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70c160ba4838414cbfc096bb3d87cbb7 is 82, key is hbase:meta,,1/info:regioninfo/1732663533394/Put/seqid=0 2024-11-26T23:25:54,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741945_1125 (size=6063) 2024-11-26T23:25:54,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741945_1125 (size=6063) 2024-11-26T23:25:54,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741945_1125 (size=6063) 2024-11-26T23:25:54,459 INFO [M:0;c707e9ace675:35437 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70c160ba4838414cbfc096bb3d87cbb7 2024-11-26T23:25:54,479 DEBUG [M:0;c707e9ace675:35437 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cb40cd06a40047e6ac19c0910c0d4781 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732663530638/Put/seqid=0 2024-11-26T23:25:54,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741946_1126 (size=7906) 2024-11-26T23:25:54,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741946_1126 (size=7906) 2024-11-26T23:25:54,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741946_1126 (size=7906) 2024-11-26T23:25:54,487 INFO [M:0;c707e9ace675:35437 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cb40cd06a40047e6ac19c0910c0d4781 2024-11-26T23:25:54,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:25:54,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46791-0x101569901eb0003, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:25:54,491 INFO [RS:2;c707e9ace675:46791 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:25:54,491 INFO [RS:2;c707e9ace675:46791 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,46791,1732663516570; zookeeper connection closed. 2024-11-26T23:25:54,491 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3cd0f7c8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3cd0f7c8 2024-11-26T23:25:54,491 INFO [M:0;c707e9ace675:35437 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cb40cd06a40047e6ac19c0910c0d4781 2024-11-26T23:25:54,492 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T23:25:54,506 DEBUG [M:0;c707e9ace675:35437 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a9aa96c7f1cc4a298f35b8d50724810a is 69, key is c707e9ace675,34663,1732663516528/rs:state/1732663517638/Put/seqid=0 2024-11-26T23:25:54,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741947_1127 (size=5445) 2024-11-26T23:25:54,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741947_1127 (size=5445) 2024-11-26T23:25:54,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741947_1127 (size=5445) 2024-11-26T23:25:54,513 INFO [M:0;c707e9ace675:35437 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a9aa96c7f1cc4a298f35b8d50724810a 2024-11-26T23:25:54,517 INFO [M:0;c707e9ace675:35437 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a9aa96c7f1cc4a298f35b8d50724810a 2024-11-26T23:25:54,518 DEBUG [M:0;c707e9ace675:35437 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/70c160ba4838414cbfc096bb3d87cbb7 as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70c160ba4838414cbfc096bb3d87cbb7 2024-11-26T23:25:54,523 INFO [M:0;c707e9ace675:35437 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/70c160ba4838414cbfc096bb3d87cbb7, entries=14, sequenceid=207, filesize=5.9 K 2024-11-26T23:25:54,524 DEBUG [M:0;c707e9ace675:35437 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cb40cd06a40047e6ac19c0910c0d4781 as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cb40cd06a40047e6ac19c0910c0d4781 2024-11-26T23:25:54,529 INFO [M:0;c707e9ace675:35437 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cb40cd06a40047e6ac19c0910c0d4781 2024-11-26T23:25:54,529 INFO [M:0;c707e9ace675:35437 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cb40cd06a40047e6ac19c0910c0d4781, entries=21, sequenceid=207, filesize=7.7 K 2024-11-26T23:25:54,530 DEBUG [M:0;c707e9ace675:35437 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a9aa96c7f1cc4a298f35b8d50724810a as hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a9aa96c7f1cc4a298f35b8d50724810a 2024-11-26T23:25:54,534 INFO [M:0;c707e9ace675:35437 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a9aa96c7f1cc4a298f35b8d50724810a 2024-11-26T23:25:54,534 INFO [M:0;c707e9ace675:35437 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45925/user/jenkins/test-data/71342a19-41c0-4459-6645-f769a57c559e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a9aa96c7f1cc4a298f35b8d50724810a, entries=3, sequenceid=207, filesize=5.3 K 2024-11-26T23:25:54,535 INFO [M:0;c707e9ace675:35437 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 102ms, sequenceid=207, compaction requested=false 2024-11-26T23:25:54,537 INFO [M:0;c707e9ace675:35437 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:25:54,537 DEBUG [M:0;c707e9ace675:35437 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663554433Disabling compacts and flushes for region at 1732663554433Disabling writes for close at 1732663554433Obtaining lock to block concurrent updates at 1732663554433Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732663554433Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1732663554433Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732663554434 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732663554434Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732663554451 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732663554451Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732663554463 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732663554478 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732663554478Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732663554492 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732663554505 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732663554505Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b655ad4: reopening flushed file at 1732663554517 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14a25312: reopening flushed file at 1732663554524 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@349f9e3a: reopening flushed file at 1732663554529 (+5 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 102ms, sequenceid=207, compaction requested=false at 1732663554535 (+6 ms)Writing region close event to WAL at 1732663554536 (+1 ms)Closed at 1732663554536 2024-11-26T23:25:54,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37013 is added to blk_1073741830_1006 (size=99424) 2024-11-26T23:25:54,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741830_1006 (size=99424) 2024-11-26T23:25:54,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43863 is added to blk_1073741830_1006 (size=99424) 2024-11-26T23:25:54,540 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:25:54,540 INFO [M:0;c707e9ace675:35437 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T23:25:54,540 INFO [M:0;c707e9ace675:35437 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35437 2024-11-26T23:25:54,540 INFO [M:0;c707e9ace675:35437 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:25:54,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:25:54,642 INFO [M:0;c707e9ace675:35437 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:25:54,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35437-0x101569901eb0000, quorum=127.0.0.1:51163, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:25:54,649 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663543975 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663543975 (inode 16688) Holder DFSClient_NONMAPREDUCE_1750284896_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732663534745/wal.1732663543975 (inode 16688) Holder DFSClient_NONMAPREDUCE_1750284896_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T23:25:54,651 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534590 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534590 (inode 16665) Holder DFSClient_NONMAPREDUCE_1750284896_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732663534297/wal.1732663534590 (inode 16665) Holder DFSClient_NONMAPREDUCE_1750284896_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T23:25:54,652 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732663525819/wal.1732663525910 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:54,654 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663545012 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663545012 (inode 16714) Holder DFSClient_NONMAPREDUCE_1750284896_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732663544568/wal.1732663545012 (inode 16714) Holder DFSClient_NONMAPREDUCE_1750284896_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T23:25:54,655 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732663545170/wal.1732663545246 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:54,655 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732663549597/wal.1732663549784 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:54,655 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732663519509/wal.1732663519582 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:25:54,657 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal.1732663519371 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal.1732663519371 (inode 16485) Holder DFSClient_NONMAPREDUCE_1750284896_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732663519191/wal.1732663519371 (inode 16485) Holder DFSClient_NONMAPREDUCE_1750284896_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T23:25:54,659 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal.1732663518958 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal.1732663518958 (inode 16462) Holder DFSClient_NONMAPREDUCE_1750284896_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732663518730/wal.1732663518958 (inode 16462) Holder DFSClient_NONMAPREDUCE_1750284896_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-26T23:25:54,663 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:25:54,666 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:25:54,666 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:25:54,666 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:25:54,666 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/hadoop.log.dir/,STOPPED} 2024-11-26T23:25:54,669 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:25:54,669 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:25:54,669 WARN [BP-1198857476-172.17.0.2-1732663512723 heartbeating to localhost/127.0.0.1:45925 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:25:54,669 WARN [BP-1198857476-172.17.0.2-1732663512723 heartbeating to localhost/127.0.0.1:45925 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1198857476-172.17.0.2-1732663512723 (Datanode Uuid 67a5b5e6-7d3a-49aa-9759-234547f3d9d4) service to localhost/127.0.0.1:45925 2024-11-26T23:25:54,670 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data5/current/BP-1198857476-172.17.0.2-1732663512723 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:25:54,670 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data6/current/BP-1198857476-172.17.0.2-1732663512723 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:25:54,671 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:25:54,673 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:25:54,673 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:25:54,673 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:25:54,673 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:25:54,673 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/hadoop.log.dir/,STOPPED} 2024-11-26T23:25:54,675 WARN [BP-1198857476-172.17.0.2-1732663512723 heartbeating to localhost/127.0.0.1:45925 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:25:54,675 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:25:54,675 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:25:54,675 WARN [BP-1198857476-172.17.0.2-1732663512723 heartbeating to localhost/127.0.0.1:45925 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1198857476-172.17.0.2-1732663512723 (Datanode Uuid 74493783-2291-407c-a260-5e7b97ce8694) service to localhost/127.0.0.1:45925 2024-11-26T23:25:54,675 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data3/current/BP-1198857476-172.17.0.2-1732663512723 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:25:54,676 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data4/current/BP-1198857476-172.17.0.2-1732663512723 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:25:54,676 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:25:54,678 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:25:54,678 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:25:54,678 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:25:54,678 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:25:54,678 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/hadoop.log.dir/,STOPPED} 2024-11-26T23:25:54,679 WARN [BP-1198857476-172.17.0.2-1732663512723 heartbeating to localhost/127.0.0.1:45925 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:25:54,680 WARN [BP-1198857476-172.17.0.2-1732663512723 heartbeating to localhost/127.0.0.1:45925 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1198857476-172.17.0.2-1732663512723 (Datanode Uuid b612efb3-28bb-4f5f-b069-259610adb432) service to localhost/127.0.0.1:45925 2024-11-26T23:25:54,680 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:25:54,680 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:25:54,680 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data1/current/BP-1198857476-172.17.0.2-1732663512723 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:25:54,680 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/cluster_12bf5d4b-ac55-8728-68fd-1f6d683c5351/data/data2/current/BP-1198857476-172.17.0.2-1732663512723 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:25:54,681 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:25:54,687 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:25:54,687 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:25:54,687 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:25:54,688 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:25:54,688 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/16119bc6-ca84-2b83-3e3e-5a3dcdffd4be/hadoop.log.dir/,STOPPED} 2024-11-26T23:25:54,697 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T23:25:54,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down